From pypy.commits at gmail.com  Tue Oct  1 05:14:57 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 01 Oct 2019 02:14:57 -0700 (PDT)
Subject: [pypy-commit] pypy record-known-result: speed up checking the
 result of unicode.find
Message-ID: <5d931911.1c69fb81.ab189.d1b4@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: record-known-result
Changeset: r97691:681aab475a24
Date: 2019-09-30 21:28 +0200
http://bitbucket.org/pypy/pypy/changeset/681aab475a24/

Log:	speed up checking the result of unicode.find

diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -1007,18 +1007,14 @@
 
         if forward:
             res_index = self._utf8.find(w_sub._utf8, start_index, end_index)
-            if res_index < 0:
-                return None
-            res = self._byte_to_index(res_index)
-            assert res >= 0
-            return space.newint(res)
         else:
             res_index = self._utf8.rfind(w_sub._utf8, start_index, end_index)
-            if res_index < 0:
-                return None
-            res = self._byte_to_index(res_index)
-            assert res >= 0
-            return space.newint(res)
+        if res_index < 0:
+            return None
+        res = self._byte_to_index(res_index)
+        assert res >= 0
+        jit.promote(res >= 0)  # always true!
+        return space.newint(res)
 
     def _unwrap_and_compute_idx_params(self, space, w_start, w_end):
         # unwrap start and stop indices, optimized for the case where

From pypy.commits at gmail.com  Tue Oct  1 05:14:59 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 01 Oct 2019 02:14:59 -0700 (PDT)
Subject: [pypy-commit] pypy record-known-result: another
 record_known_result: the content of W_UnicodeObject is known to be a valid
 utf8 string
Message-ID: <5d931913.1c69fb81.4a5d6.3cea@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: record-known-result
Changeset: r97692:298551521a23
Date: 2019-10-01 10:52 +0200
http://bitbucket.org/pypy/pypy/changeset/298551521a23/

Log:	another record_known_result: the content of W_UnicodeObject is known
	to be a valid utf8 string

diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -102,6 +102,7 @@
         return space.text_w(encode_object(space, self, 'ascii', 'strict'))
 
     def utf8_w(self, space):
+        jit.record_known_result(self._length, rutf8._check_utf8, self._utf8, True, 0, -1)
         return self._utf8
 
     def readbuf_w(self, space):
@@ -1012,8 +1013,8 @@
         if res_index < 0:
             return None
         res = self._byte_to_index(res_index)
+        jit.promote(res >= 0)  # always true!
         assert res >= 0
-        jit.promote(res >= 0)  # always true!
         return space.newint(res)
 
     def _unwrap_and_compute_idx_params(self, space, w_start, w_end):

From pypy.commits at gmail.com  Tue Oct  1 15:12:21 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 01 Oct 2019 12:12:21 -0700 (PDT)
Subject: [pypy-commit] pypy default: don't turn ord(u) and u.islower() etc
 into bridges
Message-ID: <5d93a515.1c69fb81.b79f3.a300@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r97694:f0d037751b82
Date: 2019-10-01 21:02 +0200
http://bitbucket.org/pypy/pypy/changeset/f0d037751b82/

Log:	don't turn ord(u) and u.islower() etc into bridges

diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -41,6 +41,10 @@
 def prev_codepoint_pos_dont_look_inside(utf8, p):
     return rutf8.prev_codepoint_pos(utf8, p)
 
+ at jit.elidable
+def codepoint_at_pos_dont_look_inside(utf8, p):
+    return rutf8.codepoint_at_pos(utf8, p)
+
 
 class W_UnicodeObject(W_Root):
     import_from_mixin(StringMethods)
@@ -134,7 +138,7 @@
             raise oefmt(space.w_TypeError,
                          "ord() expected a character, but string of length %d "
                          "found", self._len())
-        return space.newint(rutf8.codepoint_at_pos(self._utf8, 0))
+        return space.newint(self.codepoint_at_pos_dont_look_inside(0))
 
     def _empty(self):
         return W_UnicodeObject.EMPTY
@@ -432,7 +436,7 @@
         if self._length == 0:
             return space.w_False
         if self._length == 1:
-            return space.newbool(func(rutf8.codepoint_at_pos(self._utf8, 0)))
+            return space.newbool(func(self.codepoint_at_pos_dont_look_inside(0)))
         else:
             return self._is_generic_loop(space, self._utf8, func_name)
 
@@ -977,6 +981,11 @@
             return pos - 1
         return prev_codepoint_pos_dont_look_inside(self._utf8, pos)
 
+    def codepoint_at_pos_dont_look_inside(self, pos):
+        if self.is_ascii():
+            return ord(self._utf8[pos])
+        return codepoint_at_pos_dont_look_inside(self._utf8, pos)
+
     @always_inline
     def _unwrap_and_search(self, space, w_sub, w_start, w_end, forward=True):
         w_sub = self.convert_arg_to_w_unicode(space, w_sub)

From pypy.commits at gmail.com  Tue Oct  1 15:12:59 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 01 Oct 2019 12:12:59 -0700 (PDT)
Subject: [pypy-commit] pypy record-known-result: (cfbolz porting,
 yodada did the actual work) implement record_exact_value by
Message-ID: <5d93a53b.1c69fb81.b83e8.95c0@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: record-known-result
Changeset: r97695:0f0cf3af2deb
Date: 2019-10-01 13:47 +0200
http://bitbucket.org/pypy/pypy/changeset/0f0cf3af2deb/

Log:	(cfbolz porting, yodada did the actual work) implement
	record_exact_value by stealing the code from the record-exact-value
	branch. it's another rlib.jit.record_* hint, this one can be used to
	tell the JIT about a known constant value

diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -309,6 +309,9 @@
             op1 = [op1, SpaceOperation('-live-', [], None)]
         return op1
 
+    def rewrite_op_jit_record_exact_value(self, op):
+        return SpaceOperation("record_exact_value", [op.args[0], op.args[1]], None)
+
     def rewrite_op_debug_assert_not_none(self, op):
         if isinstance(op.args[0], Variable):
             return SpaceOperation('assert_not_none', [op.args[0]], None)
diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -602,6 +602,10 @@
                                      calldescr):
         pass
 
+    @arguments("r", "r")
+    def bhimpl_record_exact_value(a, b):
+        pass
+
     @arguments("i", returns="i")
     def bhimpl_int_copy(a):
         return a
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -541,6 +541,12 @@
         self.make_constant_class(op.getarg(0), expectedclassbox,
                                  update_last_guard=False)
 
+    def optimize_RECORD_EXACT_VALUE(self, op):
+        box = op.getarg(0)
+        expectedconstbox = op.getarg(1)
+        assert isinstance(expectedconstbox, Const)
+        self.make_constant(box, expectedconstbox)
+
     def optimize_GUARD_CLASS(self, op):
         expectedclassbox = op.getarg(1)
         info = self.ensure_ptr_info_arg0(op)
diff --git a/rpython/jit/metainterp/optimizeopt/simplify.py b/rpython/jit/metainterp/optimizeopt/simplify.py
--- a/rpython/jit/metainterp/optimizeopt/simplify.py
+++ b/rpython/jit/metainterp/optimizeopt/simplify.py
@@ -43,6 +43,9 @@
     def optimize_RECORD_EXACT_CLASS(self, op):
         pass
 
+    def optimize_RECORD_EXACT_VALUE(self, op):
+        pass
+
     def optimize_GUARD_FUTURE_CONDITION(self, op):
         self.optimizer.notice_guard_future_condition(op)
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -7212,6 +7212,21 @@
         self.optimize_loop(ops, expected)
 
 
+    def test_record_exact_value(self):
+        ops = """
+        [p0]
+        record_exact_value(p0, ConstPtr(myptr3))
+        i1 = getfield_gc_i(p0, descr=valuedescr3)
+        escape_i(i1)
+        jump(p0)
+        """
+        expected = """
+        []
+        escape_i(7)
+        jump()
+        """
+        self.optimize_loop(ops, expected)
+
     def test_quasi_immut(self):
         ops = """
         [p0, p1, i0]
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -313,6 +313,21 @@
         # an operation
         self.metainterp._record_helper_nonpure_varargs(rop.RECORD_KNOWN_RESULT, None, calldescr, allboxes)
 
+    @arguments("box", "box")
+    def opimpl_record_exact_value(self, box, const_box):
+        if isinstance(const_box, Const):
+            self.execute(rop.RECORD_EXACT_VALUE, box, const_box)
+        elif have_debug_prints():
+            if len(self.metainterp.framestack) >= 2:
+                # caller of ll_record_exact_value
+                name = self.metainterp.framestack[-2].jitcode.name
+            else:
+                name = self.jitcode.name
+            loc = self.metainterp.jitdriver_sd.warmstate.get_location_str(self.greenkey)
+            debug_print("record_exact_value with non-constant second argument, ignored",
+                    name, loc)
+
+
     @arguments("box")
     def _opimpl_any_return(self, box):
         self.metainterp.finishframe(box)
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -137,7 +137,7 @@
         return 'v' + str(num)
     if hasattr(self, '_vec_debug_info'):
         vecinfo = self._vec_debug_info
-        count = vecinfo.count 
+        count = vecinfo.count
         datatype = vecinfo.datatype
         bytesize = vecinfo.bytesize
     elif self.vector == -2:
@@ -322,7 +322,7 @@
         "shallow copy: the returned operation is meant to be used in place of self"
         # XXX specialize
         from rpython.jit.metainterp.history import DONT_CHANGE
-        
+
         if args is None:
             args = self.getarglist_copy()
         if descr is None:
@@ -1143,6 +1143,7 @@
     'QUASIIMMUT_FIELD/1d/n',    # [objptr], descr=SlowMutateDescr
     'ASSERT_NOT_NONE/1/n',      # [objptr]
     'RECORD_EXACT_CLASS/2/n',   # [objptr, clsptr]
+    'RECORD_EXACT_VALUE/2/n',   # [objptr, objptr]
     'KEEPALIVE/1/n',
     'SAVE_EXCEPTION/0/r',
     'SAVE_EXC_CLASS/0/i',       # XXX kill me
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -2,7 +2,6 @@
 import sys
 
 import py
-import weakref
 
 from rpython.rlib import rgc
 from rpython.jit.codewriter.policy import StopAtXPolicy
@@ -14,7 +13,8 @@
 from rpython.rlib.jit import (JitDriver, we_are_jitted, hint, dont_look_inside,
     loop_invariant, elidable, promote, jit_debug, assert_green,
     AssertGreenFailed, unroll_safe, current_trace_length, look_inside_iff,
-    isconstant, isvirtual, set_param, record_exact_class, record_known_result)
+    isconstant, isvirtual, set_param, record_exact_class, record_known_result,
+    record_exact_value)
 from rpython.rlib.longlong2float import float2longlong, longlong2float
 from rpython.rlib.rarithmetic import ovfcheck, is_valid_int, int_force_ge_zero
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -4016,6 +4016,33 @@
         assert res == main(10)
         self.check_resops(call_i=2)  # two calls to f, both get removed by the backend
 
+
+    def test_record_exact_value(self):
+        class A(object):
+            _immutable_fields_ = ['x']
+
+        a = A()
+        b = A()
+        a.x = 42
+        b.x = 233
+
+        @dont_look_inside
+        def make(x):
+            if x > 0:
+                return a
+            else:
+                return b
+        def f(x):
+            inst = make(x)
+            if x > 0:
+                record_exact_value(inst, a)
+            else:
+                record_exact_value(inst, b)
+            return inst.x
+        res = self.interp_operations(f, [1])
+        assert res == 42
+        self.check_operations_history(record_exact_value=1)
+
     def test_generator(self):
         def g(n):
             yield n+1
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -1232,6 +1232,36 @@
         return hop.genop("jit_record_known_result", args_v, resulttype=lltype.Void)
 
 
+def record_exact_value(value, const_value):
+    """
+    Assure the JIT that value is the same as const_value
+    """
+    assert value is const_value
+
+def ll_record_exact_value(ll_value, ll_const_value):
+    from rpython.rlib.debug import ll_assert
+    from rpython.rtyper.lltypesystem.lloperation import llop
+    from rpython.rtyper.lltypesystem import lltype
+    ll_assert(ll_value is ll_const_value, "record_exact_value called with two different arguments")
+    llop.jit_record_exact_value(lltype.Void, ll_value, ll_const_value)
+
+class Entry(ExtRegistryEntry):
+    _about_ = record_exact_value
+
+    def compute_result_annotation(self, s_inst, s_const_inst):
+        from rpython.annotator import model as annmodel
+        assert isinstance(s_inst, annmodel.SomeInstance)
+        assert isinstance(s_const_inst, annmodel.SomeInstance)
+
+    def specialize_call(self, hop):
+        from rpython.rtyper.lltypesystem import lltype
+        from rpython.rtyper import rclass
+
+        v_inst = hop.inputarg(hop.args_r[0], arg=0)
+        v_const_inst = hop.inputarg(hop.args_r[1], arg=1)
+        hop.exception_is_here()
+        return hop.gendirectcall(ll_record_exact_value, v_inst, v_const_inst)
+
 def _jit_conditional_call(condition, function, *args):
     pass           # special-cased below
 
diff --git a/rpython/rlib/test/test_jit.py b/rpython/rlib/test/test_jit.py
--- a/rpython/rlib/test/test_jit.py
+++ b/rpython/rlib/test/test_jit.py
@@ -5,7 +5,8 @@
 from rpython.rlib.jit import (hint, we_are_jitted, JitDriver, elidable_promote,
     JitHintError, oopspec, isconstant, conditional_call,
     elidable, unroll_safe, dont_look_inside, conditional_call_elidable,
-    enter_portal_frame, leave_portal_frame, record_known_result)
+    enter_portal_frame, leave_portal_frame, record_known_result,
+    record_exact_value)
 from rpython.rlib.rarithmetic import r_uint
 from rpython.rtyper.test.tool import BaseRtypingTest
 from rpython.rtyper.lltypesystem import lltype
@@ -121,6 +122,15 @@
         def f():
             pass
 
+def test_record_exact_value():
+    class A(object):
+        pass
+    a = A()
+    b = A()
+    record_exact_value(a,a) # assume not crash
+    with pytest.raises(AssertionError):
+        record_exact_value(a, b)
+
 class TestJIT(BaseRtypingTest):
     def test_hint(self):
         def f():
@@ -351,6 +361,15 @@
         t = Translation(g, [])
         t.compile_c() # does not crash
 
+    def test_record_exact_value(self):
+        class A(object):
+            pass
+        def g():
+            a = A()
+            b = A()
+            record_exact_value(a,a) # assume not crash
+        self.interpret(g, [])
+
     def test_record_known_result(self):
         from rpython.translator.interactive import Translation
         @elidable
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -563,6 +563,9 @@
     def op_jit_record_exact_class(self, *args):
         pass
 
+    def op_jit_record_exact_value(self, *args):
+        pass
+
     def op_jit_conditional_call(self, *args):
         raise NotImplementedError("should not be called while not jitted")
 
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -456,6 +456,7 @@
     'jit_is_virtual':       LLOp(canrun=True),
     'jit_force_quasi_immutable': LLOp(canrun=True),
     'jit_record_exact_class'  : LLOp(canrun=True),
+    'jit_record_exact_value'  : LLOp(canrun=True),
     'jit_ffi_save_result':  LLOp(canrun=True),
     'jit_conditional_call': LLOp(),
     'jit_conditional_call_value': LLOp(),
diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py
--- a/rpython/rtyper/lltypesystem/opimpl.py
+++ b/rpython/rtyper/lltypesystem/opimpl.py
@@ -636,6 +636,9 @@
 def op_jit_record_exact_class(x, y):
     pass
 
+def op_jit_record_exact_value(x, y):
+    pass
+
 def op_jit_ffi_save_result(*args):
     pass
 
diff --git a/rpython/translator/c/src/support.h b/rpython/translator/c/src/support.h
--- a/rpython/translator/c/src/support.h
+++ b/rpython/translator/c/src/support.h
@@ -7,6 +7,7 @@
 
 #define RUNNING_ON_LLINTERP	0
 #define OP_JIT_RECORD_EXACT_CLASS(i, c, r)  /* nothing */
+#define OP_JIT_RECORD_EXACT_VALUE(i, c, r)  /* nothing */
 
 #define FAIL_OVF(msg) _RPyRaiseSimpleException(RPyExc_OverflowError)
 

From pypy.commits at gmail.com  Tue Oct  1 15:13:01 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 01 Oct 2019 12:13:01 -0700 (PDT)
Subject: [pypy-commit] pypy record-known-result: support for integer values
 in record_exact_value
Message-ID: <5d93a53d.1c69fb81.3125f.4daf@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: record-known-result
Changeset: r97696:d6d92c45bc0b
Date: 2019-10-01 15:20 +0200
http://bitbucket.org/pypy/pypy/changeset/d6d92c45bc0b/

Log:	support for integer values in record_exact_value

diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -310,7 +310,13 @@
         return op1
 
     def rewrite_op_jit_record_exact_value(self, op):
-        return SpaceOperation("record_exact_value", [op.args[0], op.args[1]], None)
+        if getkind(op.args[0].concretetype) == "int":
+            assert getkind(op.args[1].concretetype) == "int"
+            return SpaceOperation("record_exact_value_i", [op.args[0], op.args[1]], None)
+        else:
+            assert getkind(op.args[0].concretetype) == "ref"
+            assert getkind(op.args[1].concretetype) == "ref"
+            return SpaceOperation("record_exact_value_r", [op.args[0], op.args[1]], None)
 
     def rewrite_op_debug_assert_not_none(self, op):
         if isinstance(op.args[0], Variable):
diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -603,7 +603,11 @@
         pass
 
     @arguments("r", "r")
-    def bhimpl_record_exact_value(a, b):
+    def bhimpl_record_exact_value_r(a, b):
+        pass
+
+    @arguments("i", "i")
+    def bhimpl_record_exact_value_i(a, b):
         pass
 
     @arguments("i", returns="i")
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -541,12 +541,15 @@
         self.make_constant_class(op.getarg(0), expectedclassbox,
                                  update_last_guard=False)
 
-    def optimize_RECORD_EXACT_VALUE(self, op):
+    def optimize_record_exact_value(self, op):
         box = op.getarg(0)
         expectedconstbox = op.getarg(1)
         assert isinstance(expectedconstbox, Const)
         self.make_constant(box, expectedconstbox)
 
+    optimize_RECORD_EXACT_VALUE_R = optimize_record_exact_value
+    optimize_RECORD_EXACT_VALUE_I = optimize_record_exact_value
+
     def optimize_GUARD_CLASS(self, op):
         expectedclassbox = op.getarg(1)
         info = self.ensure_ptr_info_arg0(op)
diff --git a/rpython/jit/metainterp/optimizeopt/simplify.py b/rpython/jit/metainterp/optimizeopt/simplify.py
--- a/rpython/jit/metainterp/optimizeopt/simplify.py
+++ b/rpython/jit/metainterp/optimizeopt/simplify.py
@@ -43,7 +43,10 @@
     def optimize_RECORD_EXACT_CLASS(self, op):
         pass
 
-    def optimize_RECORD_EXACT_VALUE(self, op):
+    def optimize_RECORD_EXACT_VALUE_R(self, op):
+        pass
+
+    def optimize_RECORD_EXACT_VALUE_I(self, op):
         pass
 
     def optimize_GUARD_FUTURE_CONDITION(self, op):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -7215,7 +7215,7 @@
     def test_record_exact_value(self):
         ops = """
         [p0]
-        record_exact_value(p0, ConstPtr(myptr3))
+        record_exact_value_r(p0, ConstPtr(myptr3))
         i1 = getfield_gc_i(p0, descr=valuedescr3)
         escape_i(i1)
         jump(p0)
@@ -7227,6 +7227,20 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_record_exact_value_int(self):
+        ops = """
+        [i0]
+        record_exact_value_i(i0, 15)
+        escape_i(i0)
+        jump(i0)
+        """
+        expected = """
+        []
+        escape_i(15)
+        jump()
+        """
+        self.optimize_loop(ops, expected)
+
     def test_quasi_immut(self):
         ops = """
         [p0, p1, i0]
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -314,9 +314,15 @@
         self.metainterp._record_helper_nonpure_varargs(rop.RECORD_KNOWN_RESULT, None, calldescr, allboxes)
 
     @arguments("box", "box")
-    def opimpl_record_exact_value(self, box, const_box):
+    def opimpl_record_exact_value_r(self, box, const_box):
+        return self._record_exact_value(rop.RECORD_EXACT_VALUE_R, box, const_box)
+    @arguments("box", "box")
+    def opimpl_record_exact_value_i(self, box, const_box):
+        return self._record_exact_value(rop.RECORD_EXACT_VALUE_I, box, const_box)
+
+    def _record_exact_value(self, opname, box, const_box):
         if isinstance(const_box, Const):
-            self.execute(rop.RECORD_EXACT_VALUE, box, const_box)
+            self.execute(opname, box, const_box)
         elif have_debug_prints():
             if len(self.metainterp.framestack) >= 2:
                 # caller of ll_record_exact_value
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -1143,7 +1143,8 @@
     'QUASIIMMUT_FIELD/1d/n',    # [objptr], descr=SlowMutateDescr
     'ASSERT_NOT_NONE/1/n',      # [objptr]
     'RECORD_EXACT_CLASS/2/n',   # [objptr, clsptr]
-    'RECORD_EXACT_VALUE/2/n',   # [objptr, objptr]
+    'RECORD_EXACT_VALUE_R/2/n',   # [objptr, objptr]
+    'RECORD_EXACT_VALUE_I/2/n',   # [int, int]
     'KEEPALIVE/1/n',
     'SAVE_EXCEPTION/0/r',
     'SAVE_EXC_CLASS/0/i',       # XXX kill me
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4041,7 +4041,22 @@
             return inst.x
         res = self.interp_operations(f, [1])
         assert res == 42
-        self.check_operations_history(record_exact_value=1)
+        self.check_operations_history(record_exact_value_r=1)
+
+    def test_record_exact_value_int(self):
+        @dont_look_inside
+        def make(x):
+            if x > 0:
+                return 16
+            else:
+                return x
+        def f(x):
+            y = make(x)
+            record_exact_value(y >= 0, True)
+            return y < 0
+        res = self.interp_operations(f, [1])
+        assert res == 0
+        self.check_operations_history(record_exact_value_i=1)
 
     def test_generator(self):
         def g(n):
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -1236,22 +1236,21 @@
     """
     Assure the JIT that value is the same as const_value
     """
-    assert value is const_value
+    assert value == const_value
 
 def ll_record_exact_value(ll_value, ll_const_value):
     from rpython.rlib.debug import ll_assert
     from rpython.rtyper.lltypesystem.lloperation import llop
     from rpython.rtyper.lltypesystem import lltype
-    ll_assert(ll_value is ll_const_value, "record_exact_value called with two different arguments")
+    ll_assert(ll_value == ll_const_value, "record_exact_value called with two different arguments")
     llop.jit_record_exact_value(lltype.Void, ll_value, ll_const_value)
 
 class Entry(ExtRegistryEntry):
     _about_ = record_exact_value
 
-    def compute_result_annotation(self, s_inst, s_const_inst):
+    def compute_result_annotation(self, s_val, s_const_val):
         from rpython.annotator import model as annmodel
-        assert isinstance(s_inst, annmodel.SomeInstance)
-        assert isinstance(s_const_inst, annmodel.SomeInstance)
+        annmodel.unionof(s_val, s_const_val) # produce error if types are incompatible
 
     def specialize_call(self, hop):
         from rpython.rtyper.lltypesystem import lltype
diff --git a/rpython/rlib/test/test_jit.py b/rpython/rlib/test/test_jit.py
--- a/rpython/rlib/test/test_jit.py
+++ b/rpython/rlib/test/test_jit.py
@@ -367,7 +367,16 @@
         def g():
             a = A()
             b = A()
-            record_exact_value(a,a) # assume not crash
+            record_exact_value(a, a) # assume not crash
+        self.interpret(g, [])
+
+    def test_record_exact_value_int(self):
+        @dont_look_inside
+        def f():
+            return 4
+        def g():
+            a = f()
+            record_exact_value(a >= 0, True) # does not crash
         self.interpret(g, [])
 
     def test_record_known_result(self):

From pypy.commits at gmail.com  Tue Oct  1 15:13:03 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 01 Oct 2019 12:13:03 -0700 (PDT)
Subject: [pypy-commit] pypy record-known-result: tell the jit that the
 result of str.find is always smaller than the length of
Message-ID: <5d93a53f.1c69fb81.a9b76.1df6@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: record-known-result
Changeset: r97697:0085d3e74e02
Date: 2019-10-01 15:51 +0200
http://bitbucket.org/pypy/pypy/changeset/0085d3e74e02/

Log:	tell the jit that the result of str.find is always smaller than the
	length of the string

diff --git a/rpython/jit/metainterp/test/test_string.py b/rpython/jit/metainterp/test/test_string.py
--- a/rpython/jit/metainterp/test/test_string.py
+++ b/rpython/jit/metainterp/test/test_string.py
@@ -990,7 +990,48 @@
             return len(d[s])
         assert self.interp_operations(f, [222]) == 6
 
-    def test_codepoint_index_at_byte_position_record_known_result(self):
+    def test_str_find(self):
+        jitdriver = JitDriver(greens=['x'], reds=['z', 'res'])
+        s1 = "aaaaaaef"
+        s2 = "aaaaaaaaaaaaaaaaaaaaeebbbbbbbbbbbbbbbbbbef"
+
+        @dont_look_inside
+        def pick(x):
+            if x > 0:
+                return s1
+            else:
+                return s2
+
+        @dont_look_inside
+        def pick_s(z):
+            if z < 5:
+                return "e"
+            return "ef"
+
+        def f(x):
+            z = 0
+            res = 0
+            while z < 10:
+                jitdriver.jit_merge_point(x=x, res=res, z=z)
+                s = pick(x)
+                # the following lines emulate unicode.find
+                byteindex = s.find(pick_s(x))
+                if byteindex < 0:
+                    return -1001
+                if byteindex >= len(s):  # no guard
+                    return -106
+                res += ord(s[byteindex])
+                z += 1
+            return res
+        res = self.meta_interp(f, [1], backendopt=True)
+        assert res == f(1)
+        self.check_simple_loop(guard_false=1)  # only one guard, to check for -1
+        res = self.meta_interp(f, [10], backendopt=True)
+        assert res == f(10)
+        # one guard to check whether the search string is len == 1, one to check for result -1
+        self.check_simple_loop(guard_false=2)
+
+    def test_codepoint_index_at_byte_position_record_known_result_and_invariants(self):
         from rpython.rlib import rutf8
         jitdriver = JitDriver(greens=['x'], reds=['z', 'res'])
         s1 = u"abce&#252;f".encode("utf-8")
diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -721,16 +721,22 @@
     def ll_find(s1, s2, start, end):
         if start < 0:
             start = 0
-        if end > len(s1.chars):
-            end = len(s1.chars)
+        n = len(s1.chars)
+        if end > n:
+            end = n
         if end - start < 0:
             return -1
 
         m = len(s2.chars)
         if m == 1:
-            return LLHelpers.ll_find_char(s1, s2.chars[0], start, end)
+            res = LLHelpers.ll_find_char(s1, s2.chars[0], start, end)
+            jit.record_exact_value(res < end, True)
+            return res
 
-        return LLHelpers.ll_search(s1, s2, start, end, FAST_FIND)
+        res = LLHelpers.ll_search(s1, s2, start, end, FAST_FIND)
+        jit.record_exact_value(res < end, True)
+        jit.record_exact_value(res + m <= n, True)
+        return res
 
     @staticmethod
     @signature(types.any(), types.any(), types.int(), types.int(), returns=types.int())

From pypy.commits at gmail.com  Tue Oct  1 15:13:05 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 01 Oct 2019 12:13:05 -0700 (PDT)
Subject: [pypy-commit] pypy record-known-result: hints on the bounds of the
 results of some of the rutf8 functions
Message-ID: <5d93a541.1c69fb81.f850b.ba97@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: record-known-result
Changeset: r97698:7151aca5645a
Date: 2019-10-01 15:53 +0200
http://bitbucket.org/pypy/pypy/changeset/7151aca5645a/

Log:	hints on the bounds of the results of some of the rutf8 functions

diff --git a/dotviewer/graphclient.py b/dotviewer/graphclient.py
--- a/dotviewer/graphclient.py
+++ b/dotviewer/graphclient.py
@@ -95,8 +95,8 @@
     except EOFError:
         ioerror = ioerror or IOError("connection unexpectedly closed "
                                      "(graphserver crash?)")
-    if ioerror is not None:
-        raise ioerror
+    #if ioerror is not None:
+    #    raise ioerror
 
 def send_error(io, e):
     try:
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -760,7 +760,7 @@
         assert stop >= 0
         byte_start = 0
         for i in range(start):
-            byte_start = next_codepoint_pos_dont_look_inside(self._utf8, byte_start)
+            byte_start = self.next_codepoint_pos_dont_look_inside(byte_start)
         byte_stop = len(self._utf8)
         for i in range(self._len() - stop):
             byte_stop = prev_codepoint_pos_dont_look_inside(self._utf8, byte_stop)
@@ -920,7 +920,7 @@
         if not self.is_ascii():
             storage = self._get_index_storage()
             jit.record_known_result(
-                end, rutf8.codepoint_position_at_index, self._utf8, storage, index + 1)
+                end, rutf8._codepoint_position_at_index, self._utf8, storage, index + 1)
         return W_UnicodeObject(self._utf8[start:end], 1)
 
     @jit.unroll_safe
@@ -952,6 +952,7 @@
         return self._length == len(self._utf8)
 
     def _index_to_byte(self, index):
+        assert 0 <= index < self._len()
         if self.is_ascii():
             assert index >= 0
             return index
@@ -974,6 +975,7 @@
         """ this returns index such that self._index_to_byte(index) == bytepos
         NB: this is slow! roughly logarithmic with a big constant
         """
+        assert 0 <= bytepos < len(self._utf8)
         if self.is_ascii():
             return bytepos
         return rutf8.codepoint_index_at_byte_position(
@@ -982,7 +984,10 @@
     def next_codepoint_pos_dont_look_inside(self, pos):
         if self.is_ascii():
             return pos + 1
-        return next_codepoint_pos_dont_look_inside(self._utf8, pos)
+        res = next_codepoint_pos_dont_look_inside(self._utf8, pos)
+        jit.record_exact_value(res >= 0, True)
+        jit.record_exact_value(res <= len(self._utf8), True)
+        return res
 
     def prev_codepoint_pos_dont_look_inside(self, pos):
         if self.is_ascii():
@@ -1013,7 +1018,6 @@
         if res_index < 0:
             return None
         res = self._byte_to_index(res_index)
-        jit.promote(res >= 0)  # always true!
         assert res >= 0
         return space.newint(res)
 
diff --git a/rpython/jit/metainterp/test/test_string.py b/rpython/jit/metainterp/test/test_string.py
--- a/rpython/jit/metainterp/test/test_string.py
+++ b/rpython/jit/metainterp/test/test_string.py
@@ -1053,9 +1053,14 @@
                 s = pick(x)
                 # the following lines emulate unicode.find
                 byteindex = s.find(search[z])
-                assert byteindex >= 0
+                if byteindex < 0:
+                    return -1001
                 storage = rutf8.create_utf8_index_storage(s, len(s) - 1)
                 index = rutf8.codepoint_index_at_byte_position(s, storage, byteindex, len(s) - 1)
+                if index < 0:  # no guard
+                    return -1000
+                if index >= len(s) - 1:  # no guard
+                    return -1004
                 # then we use the resulting codepoint index in conjunction with
                 # the string to get at a byte index
                 b = rutf8.codepoint_position_at_index(s, storage, index)
@@ -1068,5 +1073,5 @@
         f(1)
         res = self.meta_interp(f, [1], backendopt=True)
         assert res == f(1)
-        self.check_simple_loop(int_sub=1)
+        self.check_simple_loop(int_sub=1, guard_false=1)
 
diff --git a/rpython/rlib/rutf8.py b/rpython/rlib/rutf8.py
--- a/rpython/rlib/rutf8.py
+++ b/rpython/rlib/rutf8.py
@@ -545,13 +545,20 @@
         break
     return storage
 
- at jit.elidable
 def codepoint_position_at_index(utf8, storage, index):
     """ Return byte index of a character inside utf8 encoded string, given
     storage of type UTF8_INDEX_STORAGE.  The index must be smaller than
     or equal to the utf8 length: if needed, check explicitly before calling
     this function.
     """
+    res = _codepoint_position_at_index(utf8, storage, index)
+    # tell the jit about the invariants of the result
+    jit.record_exact_value(res >= 0, True)
+    jit.record_exact_value(res < len(utf8), True)
+    return res
+
+ at jit.elidable
+def _codepoint_position_at_index(utf8, storage, index):
     current = index >> 6
     ofs = ord(storage[current].ofs[(index >> 2) & 0x0F])
     bytepos = storage[current].baseindex + ofs
@@ -599,7 +606,10 @@
     is not tiny either.
     """
     res = _codepoint_index_at_byte_position(utf8, storage, bytepos, num_codepoints)
-    jit.record_known_result(bytepos, codepoint_position_at_index, utf8, storage, res)
+    jit.record_known_result(bytepos, _codepoint_position_at_index, utf8, storage, res)
+    # tell the JIT that there are no bounds checks needed on the resulting indices
+    jit.record_exact_value(res >= 0, True)
+    jit.record_exact_value(res < num_codepoints, True)
     return res
 
 @jit.elidable

From pypy.commits at gmail.com  Tue Oct  1 15:13:07 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 01 Oct 2019 12:13:07 -0700 (PDT)
Subject: [pypy-commit] pypy record-known-result: merge default
Message-ID: <5d93a543.1c69fb81.2acc6.fe28@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: record-known-result
Changeset: r97699:d07336d23a84
Date: 2019-10-01 21:12 +0200
http://bitbucket.org/pypy/pypy/changeset/d07336d23a84/

Log:	merge default

diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -41,6 +41,10 @@
 def prev_codepoint_pos_dont_look_inside(utf8, p):
     return rutf8.prev_codepoint_pos(utf8, p)
 
+ at jit.elidable
+def codepoint_at_pos_dont_look_inside(utf8, p):
+    return rutf8.codepoint_at_pos(utf8, p)
+
 
 class W_UnicodeObject(W_Root):
     import_from_mixin(StringMethods)
@@ -135,7 +139,7 @@
             raise oefmt(space.w_TypeError,
                          "ord() expected a character, but string of length %d "
                          "found", self._len())
-        return space.newint(rutf8.codepoint_at_pos(self._utf8, 0))
+        return space.newint(self.codepoint_at_pos_dont_look_inside(0))
 
     def _empty(self):
         return W_UnicodeObject.EMPTY
@@ -433,7 +437,7 @@
         if self._length == 0:
             return space.w_False
         if self._length == 1:
-            return space.newbool(func(rutf8.codepoint_at_pos(self._utf8, 0)))
+            return space.newbool(func(self.codepoint_at_pos_dont_look_inside(0)))
         else:
             return self._is_generic_loop(space, self._utf8, func_name)
 
@@ -994,6 +998,11 @@
             return pos - 1
         return prev_codepoint_pos_dont_look_inside(self._utf8, pos)
 
+    def codepoint_at_pos_dont_look_inside(self, pos):
+        if self.is_ascii():
+            return ord(self._utf8[pos])
+        return codepoint_at_pos_dont_look_inside(self._utf8, pos)
+
     @always_inline
     def _unwrap_and_search(self, space, w_sub, w_start, w_end, forward=True):
         w_sub = self.convert_arg_to_w_unicode(space, w_sub)

From pypy.commits at gmail.com  Tue Oct  1 16:27:45 2019
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 01 Oct 2019 13:27:45 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: Add failing apptest related to the
 firstiter asyncgen_hook
Message-ID: <5d93b6c1.1c69fb81.692c0.f4c1@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6
Changeset: r97700:0078687570a6
Date: 2019-10-01 20:05 +0100
http://bitbucket.org/pypy/pypy/changeset/0078687570a6/

Log:	Add failing apptest related to the firstiter asyncgen_hook

diff --git a/pypy/interpreter/test/apptest_coroutine.py b/pypy/interpreter/test/apptest_coroutine.py
--- a/pypy/interpreter/test/apptest_coroutine.py
+++ b/pypy/interpreter/test/apptest_coroutine.py
@@ -1,6 +1,8 @@
 import pytest
 from pytest import raises
 
+import sys
+
 
 def test_cannot_iterate():
     async def f(x):
@@ -14,6 +16,7 @@
     class X:
         def __aiter__(self):
             return MyAIter()
+
     class MyAIter:
         async def __anext__(self):
             return 42
@@ -94,7 +97,6 @@
     assert isinstance(c.value.__cause__, ZeroDivisionError)
 
 def test_set_coroutine_wrapper():
-    import sys
     async def f():
         pass
     seen = []
@@ -610,7 +612,6 @@
 
 def test_async_aclose_in_finalize_hook_await_in_finally():
     import gc
-    import sys
     import types
 
     @types.coroutine
@@ -728,3 +729,49 @@
         assert val2.value == 2
 
     run_async(run())
+
+class suspend:
+    """
+    A simple awaitable that returns control to the "event loop" with `msg`
+    as value.
+    """
+    def __init__(self, msg=None):
+        self.msg = msg
+
+    def __await__(self):
+        yield self.msg
+
+def test_asyncgen_hooks_shutdown():
+    finalized = 0
+    asyncgens = []
+
+    def register_agen(agen):
+        asyncgens.append(agen)
+
+    async def waiter(timeout):
+        nonlocal finalized
+        try:
+            await suspend('running waiter')
+            yield 1
+        finally:
+            await suspend('closing waiter')
+            finalized += 1
+
+    async def wait():
+        async for _ in waiter(1):
+            pass
+
+    task1 = wait()
+    task2 = wait()
+    old_hooks = sys.get_asyncgen_hooks()
+    try:
+        sys.set_asyncgen_hooks(firstiter=register_agen)
+        assert task1.send(None) == 'running waiter'
+        assert task2.send(None) == 'running waiter'
+        assert len(asyncgens) == 2
+
+        assert run_async(asyncgens[0].aclose()) == (['closing waiter'], None)
+        assert run_async(asyncgens[1].aclose()) == (['closing waiter'], None)
+        assert finalized == 2
+    finally:
+        sys.set_asyncgen_hooks(*old_hooks)

From pypy.commits at gmail.com  Tue Oct  1 16:27:47 2019
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 01 Oct 2019 13:27:47 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: Use suspend() helper in more tests to
 avoid importing types from lib-python
Message-ID: <5d93b6c3.1c69fb81.26ba4.7b20@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6
Changeset: r97701:d3b503bab689
Date: 2019-10-01 21:26 +0100
http://bitbucket.org/pypy/pypy/changeset/d3b503bab689/

Log:	Use suspend() helper in more tests to avoid importing types from
	lib-python

diff --git a/pypy/interpreter/test/apptest_coroutine.py b/pypy/interpreter/test/apptest_coroutine.py
--- a/pypy/interpreter/test/apptest_coroutine.py
+++ b/pypy/interpreter/test/apptest_coroutine.py
@@ -4,6 +4,18 @@
 import sys
 
 
+class suspend:
+    """
+    A simple awaitable that returns control to the "event loop" with `msg`
+    as value.
+    """
+    def __init__(self, msg=None):
+        self.msg = msg
+
+    def __await__(self):
+        yield self.msg
+
+
 def test_cannot_iterate():
     async def f(x):
         pass
@@ -111,7 +123,6 @@
     sys.set_coroutine_wrapper(None)
     assert sys.get_coroutine_wrapper() is None
 
-
 def test_async_with():
     seen = []
     class X:
@@ -519,12 +530,6 @@
     raises(RuntimeError, run().send, None)
 
 def test_async_aclose_await_in_finally():
-    import types
-
-    @types.coroutine
-    def coro():
-        yield 'coro'
-
     state = 0
     async def ag():
         nonlocal state
@@ -532,7 +537,7 @@
             yield
         finally:
             state = 1
-            await coro()
+            await suspend('coro')
             state = 2
 
     async def run():
@@ -551,12 +556,6 @@
     assert state == 2
 
 def test_async_aclose_await_in_finally_with_exception():
-    import types
-
-    @types.coroutine
-    def coro():
-        yield 'coro'
-
     state = 0
     async def ag():
         nonlocal state
@@ -565,7 +564,7 @@
         finally:
             state = 1
             try:
-                await coro()
+                await suspend('coro')
             except Exception as exc:
                 state = exc
 
@@ -586,17 +585,12 @@
     assert state == exc
 
 def test_agen_aclose_await_and_yield_in_finally():
-    import types
-    @types.coroutine
-    def bar():
-        yield 42
-
     async def foo():
         try:
             yield 1
             1 / 0
         finally:
-            await bar()
+            await suspend(42)
             yield 12
 
     async def run():
@@ -612,12 +606,6 @@
 
 def test_async_aclose_in_finalize_hook_await_in_finally():
     import gc
-    import types
-
-    @types.coroutine
-    def coro():
-        yield 'coro'
-
     state = 0
     async def ag():
         nonlocal state
@@ -625,7 +613,7 @@
             yield
         finally:
             state = 1
-            await coro()
+            await suspend('coro')
             state = 2
 
     async def run():
@@ -730,17 +718,6 @@
 
     run_async(run())
 
-class suspend:
-    """
-    A simple awaitable that returns control to the "event loop" with `msg`
-    as value.
-    """
-    def __init__(self, msg=None):
-        self.msg = msg
-
-    def __await__(self):
-        yield self.msg
-
 def test_asyncgen_hooks_shutdown():
     finalized = 0
     asyncgens = []

From pypy.commits at gmail.com  Tue Oct  1 16:27:49 2019
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 01 Oct 2019 13:27:49 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: Use built-in module _io instead of io
Message-ID: <5d93b6c5.1c69fb81.e0627.d642@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6
Changeset: r97702:f7ddde9c4602
Date: 2019-10-01 21:26 +0100
http://bitbucket.org/pypy/pypy/changeset/f7ddde9c4602/

Log:	Use built-in module _io instead of io

diff --git a/pypy/module/sys/std_test.py b/pypy/module/sys/std_test.py
--- a/pypy/module/sys/std_test.py
+++ b/pypy/module/sys/std_test.py
@@ -1,18 +1,18 @@
 # Install standard streams for tests that don't call app_main.  Always
 # use line buffering, even for tests that capture standard descriptors.
 
-import io
+import _io
 
-stdin = io.open(0, "r", encoding="ascii",
+stdin = _io.open(0, "r", encoding="ascii",
                 closefd=False)
 stdin.buffer.raw.name = "<stdin>"
 
-stdout = io.open(1, "w", encoding="ascii",
+stdout = _io.open(1, "w", encoding="ascii",
                  buffering=1,
                  closefd=False)
 stdout.buffer.raw.name = "<stdout>"
 
-stderr = io.open(2, "w", encoding="ascii",
+stderr = _io.open(2, "w", encoding="ascii",
                  errors="backslashreplace",
                  buffering=1,
                  closefd=False)

From pypy.commits at gmail.com  Tue Oct  1 16:33:15 2019
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 01 Oct 2019 13:33:15 -0700 (PDT)
Subject: [pypy-commit] pypy py3.7: hg merge py3.6
Message-ID: <5d93b80b.1c69fb81.c9312.4227@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.7
Changeset: r97703:11316d0ce627
Date: 2019-10-01 21:32 +0100
http://bitbucket.org/pypy/pypy/changeset/11316d0ce627/

Log:	hg merge py3.6

diff too long, truncating to 2000 out of 21056 lines

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -40,11 +40,11 @@
   Armin Rigo
   Maciej Fijalkowski
   Carl Friedrich Bolz-Tereick
+  Matti Picus
   Antonio Cuni
   Amaury Forgeot d'Arc
-  Matti Picus
+  Ronan Lamy
   Samuele Pedroni
-  Ronan Lamy
   Alex Gaynor
   Philip Jenvey
   Richard Plangger
@@ -94,6 +94,7 @@
   Jason Creighton
   Mark Young
   Alex Martelli
+  Andrew Lawrence
   Spenser Bauman
   Michal Bendowski
   Jan de Mooij
@@ -106,11 +107,11 @@
   Stefan Schwarzer
   Tomek Meka
   Valentino Volonghi
+  Stefan Beyer
   Patrick Maupin
   Devin Jeanpierre
   Bob Ippolito
   Bruno Gola
-  Andrew Lawrence
   David Malcolm
   Squeaky
   Edd Barrett
@@ -124,7 +125,6 @@
   Wenzhu Man
   Konstantin Lopuhin
   John Witulski
-  Stefan Beyer
   Jeremy Thurgood
   Greg Price
   Ivan Sichmann Freitas
@@ -138,6 +138,7 @@
   Pavel Vinogradov
   William Leslie
   Pawe&#322; Piotr Przeradowski
+  Stian Andreassen
   marky1991
   Ilya Osadchiy
   Tobias Oberstein
@@ -146,14 +147,13 @@
   Taavi Burns
   Adrian Kuhn
   tav
-  Stian Andreassen
   Georg Brandl
   Joannah Nanjekye
+  Julian Berman
   Bert Freudenberg
   Wanja Saatkamp
   Mike Blume
   Gerald Klix
-  Julian Berman
   Oscar Nierstrasz
   Rami Chowdhury
   Stefan H. Muller
@@ -204,6 +204,7 @@
   Andrews Medina
   Aaron Iles
   Toby Watson
+  Lin Cheng
   Daniel Patrick
   Stuart Williams
   Antoine Pitrou
@@ -245,6 +246,7 @@
   Valentina Mukhamedzhanova
   Stefano Parmesan
   touilleMan
+  Anthony Sottile
   Marc Abramowitz
   Arjun Naik
   Aaron Gallagher
@@ -254,7 +256,6 @@
   Omer Katz
   Jacek Generowicz
   Tomasz Dziopa
-  Lin Cheng
   Sylvain Thenault
   Jakub Stasiak
   Andrew Dalke
@@ -285,7 +286,6 @@
   Lene Wagner
   Tomo Cocoa
   Miro Hron&#269;ok
-  Anthony Sottile
   David Lievens
   Neil Blakey-Milner
   Henrik Vendelbo
@@ -294,11 +294,14 @@
   Christoph Gerum
   Miguel de Val Borro
   Artur Lisiecki
+  joserubiovidales at gmail.com
   afteryu
   Toni Mattis
+  Vincent Michel
   Laurens Van Houtven
   Bobby Impollonia
   Roberto De Ioris
+  Yannick Jadoul
   Jeong YunWon
   Christopher Armstrong
   Aaron Tubbs
@@ -312,6 +315,7 @@
   Fabio Niephaus
   Akira Li
   Gustavo Niemeyer
+  joachim-ballmann at bitbucket.org
   Nate Bragg
   Lucas Stadler
   roberto at goyle
@@ -331,8 +335,12 @@
   Ben Darnell
   Juan Francisco Cantero Hurtado
   Godefroid Chappelle
+  Paul Ganssle
+  Michal Kuffa
   Stephan Busemann
+  Bystroushaak
   Dan Colish
+  Ram Rachum
   timo
   Volodymyr Vladymyrov
   Daniel Neuh&#228;user
@@ -342,18 +350,22 @@
   Chris Lambacher
   John Aldis
   coolbutuseless at gmail.com
+  Yasen Kiprov
   Mike Bayer
   Rodrigo Ara&#250;jo
   Daniil Yarancev
   Min RK
   OlivierBlanvillain
+  dakarpov at gmail.com
   Jonas Pfannschmidt
   Zearin
   Johan Forsberg
   Andrey Churin
   Dan Crosta
   reubano at gmail.com
+  Ryan Hileman
   Stanis&#322;aw Halik
+  DeVerne Jones
   Julien Phalip
   Roman Podoliaka
   Steve Papanik
@@ -369,17 +381,20 @@
   Jim Hunziker
   shoma hosaka
   Buck Golemon
+  whitequark
   Iraklis D.
   JohnDoe
   yrttyr
   Michael Chermside
   Anna Ravencroft
   remarkablerocket
+  Ivan
   Petre Vijiac
   Berker Peksag
   Christian Muirhead
   soareschen
   Matthew Miller
+  Jesdi
   Konrad Delong
   Dinu Gherman
   pizi
@@ -398,13 +413,16 @@
   Markus Unterwaditzer
   Kristoffer Kleine
   Graham Markall
+  paugier
   Dan Loewenherz
   werat
   Filip Salomonsson
   Niclas Olofsson
+  Zsolt Cserna
   Chris Pressey
   Tobias Diaz
   Paul Graydon
+  mkuffa
   Nikolaos-Digenis Karagiannis
   Kurt Griffiths
   Ben Mather
diff --git a/extra_tests/cffi_tests/cffi1/test_re_python.py b/extra_tests/cffi_tests/cffi1/test_re_python.py
--- a/extra_tests/cffi_tests/cffi1/test_re_python.py
+++ b/extra_tests/cffi_tests/cffi1/test_re_python.py
@@ -66,7 +66,7 @@
     int add43(int, ...);
     int globalvar42;
     const int globalconst42;
-    const char *const globalconsthello = "hello";
+    const char *const globalconsthello;
     int no_such_function(int);
     int no_such_globalvar;
     struct foo_s;
diff --git a/extra_tests/cffi_tests/test_egg_version.py b/extra_tests/cffi_tests/test_version.py
rename from extra_tests/cffi_tests/test_egg_version.py
rename to extra_tests/cffi_tests/test_version.py
--- a/extra_tests/cffi_tests/test_egg_version.py
+++ b/extra_tests/cffi_tests/test_version.py
@@ -1,6 +1,7 @@
 from email.parser import Parser
 
 import py
+from urllib.request import urlopen
 
 import cffi
 import pypy
@@ -10,3 +11,11 @@
 def test_egg_version():
     info = Parser().parsestr(egg_info.read())
     assert info['version'] == cffi.__version__
+
+def test_pycparser_version():
+    url = 'https://raw.githubusercontent.com/eliben/pycparser/master/pycparser/__init__.py'
+    source = urlopen(url).read().decode('utf8')
+    dest = py.path.local(__file__).join('..', '..', '..', 'lib_pypy', 'cffi',
+                                        '_pycparser', '__init__.py').read()
+    # if this fails, the vendored pycparser is not the latest version
+    assert source.strip() == dest.strip()
diff --git a/extra_tests/test_semlock.py b/extra_tests/test_semlock.py
new file mode 100644
--- /dev/null
+++ b/extra_tests/test_semlock.py
@@ -0,0 +1,36 @@
+from _multiprocessing import SemLock
+from threading import Thread
+import _thread
+import time
+
+
+def test_notify_all():
+    """A low-level variation on test_notify_all() in lib-python's
+    _test_multiprocessing.py
+    """
+    N_THREADS = 1000
+    lock = SemLock(0, 1, 1, "/test_notify_all", True)
+    results = []
+
+    def f(n):
+        if lock.acquire(timeout=5.):
+            results.append(n)
+            lock.release()
+
+    threads = [Thread(target=f, args=(i,)) for i in range(N_THREADS)]
+    n_started = N_THREADS
+    with lock:
+        for t in threads:
+            try:
+                t.start()
+            except _thread.error:
+                # too many threads for this system
+                t.started = False
+                n_started -= 1
+            else:
+                t.started = True
+        time.sleep(0.1)
+    for t in threads:
+        if t.started:
+            t.join()
+    assert len(results) == n_started
diff --git a/lib-python/3/stat.py b/lib-python/3/stat.py
--- a/lib-python/3/stat.py
+++ b/lib-python/3/stat.py
@@ -40,6 +40,10 @@
 S_IFIFO  = 0o010000  # fifo (named pipe)
 S_IFLNK  = 0o120000  # symbolic link
 S_IFSOCK = 0o140000  # socket file
+# Fallbacks for uncommon platform-specific constants
+S_IFDOOR = 0
+S_IFPORT = 0
+S_IFWHT = 0
 
 # Functions to test for each file type
 
@@ -71,6 +75,18 @@
     """Return True if mode is from a socket."""
     return S_IFMT(mode) == S_IFSOCK
 
+def S_ISDOOR(mode):
+    """Return True if mode is from a door."""
+    return False
+
+def S_ISPORT(mode):
+    """Return True if mode is from an event port."""
+    return False
+
+def S_ISWHT(mode):
+    """Return True if mode is from a whiteout."""
+    return False
+
 # Names for permission bits
 
 S_ISUID = 0o4000  # set UID bit
diff --git a/lib-python/3/test/test_concurrent_futures.py b/lib-python/3/test/test_concurrent_futures.py
--- a/lib-python/3/test/test_concurrent_futures.py
+++ b/lib-python/3/test/test_concurrent_futures.py
@@ -636,6 +636,7 @@
                 futures_list.remove(future)
                 wr = weakref.ref(future)
                 del future
+                test.support.gc_collect()
                 self.assertIsNone(wr())
 
         futures_list[0].set_result("test")
@@ -643,6 +644,7 @@
             futures_list.remove(future)
             wr = weakref.ref(future)
             del future
+            test.support.gc_collect()
             self.assertIsNone(wr())
             if futures_list:
                 futures_list[0].set_result("test")
@@ -742,6 +744,7 @@
         for obj in self.executor.map(make_dummy_object, range(10)):
             wr = weakref.ref(obj)
             del obj
+            test.support.gc_collect()
             self.assertIsNone(wr())
 
 
diff --git a/lib-python/3/test/test_dis.py b/lib-python/3/test/test_dis.py
--- a/lib-python/3/test/test_dis.py
+++ b/lib-python/3/test/test_dis.py
@@ -145,7 +145,7 @@
               1)
     pass
 
-_bug1333982 = """\
+dis_bug1333982 = """\
 %3d           0 LOAD_CONST               1 (0)
               2 POP_JUMP_IF_TRUE        26
               4 LOAD_GLOBAL              0 (AssertionError)
diff --git a/lib-python/3/test/test_peepholer.py b/lib-python/3/test/test_peepholer.py
--- a/lib-python/3/test/test_peepholer.py
+++ b/lib-python/3/test/test_peepholer.py
@@ -186,9 +186,11 @@
         code = compile('a=1<<1000', '', 'single')
         self.assertInBytecode(code, 'LOAD_CONST', 1000)
         self.assertNotIn(1<<1000, code.co_consts)
-        code = compile('a=2**1000', '', 'single')
-        self.assertInBytecode(code, 'LOAD_CONST', 1000)
-        self.assertNotIn(2**1000, code.co_consts)
+        # difference to CPython: PyPy allows slightly larger constants to be
+        # created
+        code = compile('a=2**10000', '', 'single')
+        self.assertInBytecode(code, 'LOAD_CONST', 10000)
+        self.assertNotIn(2**10000, code.co_consts)
 
     @cpython_only # we currently not bother to implement that
     def test_binary_subscr_on_unicode(self):
diff --git a/lib-python/3/test/test_readline.py b/lib-python/3/test/test_readline.py
--- a/lib-python/3/test/test_readline.py
+++ b/lib-python/3/test/test_readline.py
@@ -243,7 +243,7 @@
     #   See https://cnswww.cns.cwru.edu/php/chet/readline/CHANGES
     # - editline: history size is broken on OS X 10.11.6.
     #   Newer versions were not tested yet.
-    @unittest.skipIf(readline._READLINE_VERSION < 0x600,
+    @unittest.skipIf(getattr(readline, "_READLINE_VERSION", 0x601) < 0x600,
                      "this readline version does not support history-size")
     @unittest.skipIf(is_editline,
                      "editline history size configuration is broken")
diff --git a/lib-python/3/test/test_regrtest.py b/lib-python/3/test/test_regrtest.py
--- a/lib-python/3/test/test_regrtest.py
+++ b/lib-python/3/test/test_regrtest.py
@@ -258,7 +258,6 @@
                 self.checkError([opt, '0', '-T'], "don't go together")
                 self.checkError([opt, '0', '-T'], "don't go together")
                 self.checkError([opt, '0', '-l'], "don't go together")
-                self.checkError([opt, '0', '-M', '4G'], "don't go together")
 
     def test_coverage(self):
         for opt in '-T', '--coverage':
diff --git a/lib-python/3/test/test_threading.py b/lib-python/3/test/test_threading.py
--- a/lib-python/3/test/test_threading.py
+++ b/lib-python/3/test/test_threading.py
@@ -561,6 +561,7 @@
         self.assertEqual(err, b"")
         self.assertEqual(data, "Thread-1\nTrue\nTrue\n")
 
+    @test.support.cpython_only
     @requires_type_collecting
     def test_main_thread_during_shutdown(self):
         # bpo-31516: current_thread() should still point to the main thread
diff --git a/lib-python/3/test/test_time.py b/lib-python/3/test/test_time.py
--- a/lib-python/3/test/test_time.py
+++ b/lib-python/3/test/test_time.py
@@ -588,6 +588,8 @@
             self.skipTest('could not set locale.LC_ALL to fr_FR')
         # This should not cause an exception
         time.strftime("%B", (2009,2,1,0,0,0,0,0,0))
+        # PyPy addition:
+        time.strftime("%B", (2009,8,1,0,0,0,0,0,0)).lower()
 
 
 class _TestAsctimeYear:
diff --git a/lib-python/3/unittest/test/testmock/testhelpers.py b/lib-python/3/unittest/test/testmock/testhelpers.py
--- a/lib-python/3/unittest/test/testmock/testhelpers.py
+++ b/lib-python/3/unittest/test/testmock/testhelpers.py
@@ -3,6 +3,8 @@
 import types
 import unittest
 
+from test.support import cpython_only
+
 from unittest.mock import (
     call, _Call, create_autospec, MagicMock,
     Mock, ANY, _CallList, patch, PropertyMock, _callable
@@ -875,7 +877,7 @@
         # plain data descriptor
         check_data_descriptor(foo.desc)
 
-
+    @cpython_only # PyPy can easily extract a spec from a builtin function
     def test_autospec_on_bound_builtin_function(self):
         meth = types.MethodType(time.ctime, time.time())
         self.assertIsInstance(meth(), str)
diff --git a/lib_pypy/_curses_build.py b/lib_pypy/_curses_build.py
--- a/lib_pypy/_curses_build.py
+++ b/lib_pypy/_curses_build.py
@@ -34,6 +34,13 @@
 #define NCURSES_OPAQUE 0
 #endif
 
+
+/* ncurses 6 change behaviour  and makes all pointers opaque, 
+  lets define backward compatibility. It doesn't harm 
+  previous versions */
+
+#define NCURSES_INTERNALS 1
+#define NCURSES_REENTRANT 0
 #include <ncurses.h>
 #include <panel.h>
 #include <term.h>
diff --git a/lib_pypy/_dbm.py b/lib_pypy/_dbm.py
--- a/lib_pypy/_dbm.py
+++ b/lib_pypy/_dbm.py
@@ -149,7 +149,7 @@
     lib = CDLL("/usr/lib/libdbm.dylib") # OS X
     _platform = 'osx'
 
-library = "GNU gdbm"
+library = "Berkeley DB"
 
 funcs = {}
 _init_func('open', (c_char_p, c_int, c_int), restype=c_void_p)
diff --git a/lib_pypy/_pypy_testcapi.py b/lib_pypy/_pypy_testcapi.py
--- a/lib_pypy/_pypy_testcapi.py
+++ b/lib_pypy/_pypy_testcapi.py
@@ -61,6 +61,8 @@
     assert output_dir is not None
 
     from distutils.ccompiler import new_compiler
+    from distutils import log
+    log.set_verbosity(3)
 
     compiler = new_compiler()
     compiler.output_dir = output_dir
@@ -72,7 +74,8 @@
         ccflags = ['-fPIC', '-Wimplicit-function-declaration']
     res = compiler.compile([os.path.join(thisdir, csource)],
                            include_dirs=[include_dir],
-                           extra_preargs=ccflags)
+                           extra_preargs=ccflags,
+                          )
     object_filename = res[0]
 
     # set link options
diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -746,6 +746,8 @@
         self.__initialized = True
 
     def close(self):
+        if not self.__initialized:
+            raise ProgrammingError("Base Cursor.__init__ not called.")
         self.__connection._check_thread()
         self.__connection._check_closed()
         if self.__statement:
@@ -1043,6 +1045,7 @@
         return list(self)
 
     def __get_connection(self):
+        self.__check_cursor()
         return self.__connection
     connection = property(__get_connection)
 
diff --git a/lib_pypy/_stat.py b/lib_pypy/_stat.py
deleted file mode 100644
--- a/lib_pypy/_stat.py
+++ /dev/null
@@ -1,6 +0,0 @@
-# Assume not Solaris
-
-S_IFDOOR = 0
-
-def S_ISDOOR(mode):
-    return False
diff --git a/lib_pypy/_testcapimodule.c b/lib_pypy/_testcapimodule.c
--- a/lib_pypy/_testcapimodule.c
+++ b/lib_pypy/_testcapimodule.c
@@ -3132,6 +3132,8 @@
     return PyLong_FromLong(r);
 }
 
+#endif  /* PYPY_VERSION */
+
 static int
 check_time_rounding(int round)
 {
@@ -3192,6 +3194,8 @@
     return Py_BuildValue("Nl", _PyLong_FromTime_t(sec), nsec);
 }
 
+#ifndef PYPY_VERSION
+
 static void
 slot_tp_del(PyObject *self)
 {
@@ -4048,8 +4052,6 @@
     Py_RETURN_NONE;
 }
 
-#ifndef PYPY_VERSION
-
 static PyObject *
 test_pytime_fromseconds(PyObject *self, PyObject *args)
 {
@@ -4187,6 +4189,8 @@
     return _PyTime_AsNanosecondsObject(ms);
 }
 
+#ifndef PYPY_VERSION
+
 static PyObject*
 get_recursion_depth(PyObject *self, PyObject *args)
 {
@@ -4809,9 +4813,11 @@
     {"crash_no_current_thread", (PyCFunction)crash_no_current_thread, METH_NOARGS},
 #ifndef PYPY_VERSION
     {"run_in_subinterp",        run_in_subinterp,                METH_VARARGS},
+#endif
     {"pytime_object_to_time_t", test_pytime_object_to_time_t,  METH_VARARGS},
     {"pytime_object_to_timeval", test_pytime_object_to_timeval,  METH_VARARGS},
     {"pytime_object_to_timespec", test_pytime_object_to_timespec,  METH_VARARGS},
+#ifndef PYPY_VERSION
     {"with_tp_del",             with_tp_del,                     METH_VARARGS},
 #endif
     {"create_cfunction",        create_cfunction,                METH_NOARGS},
@@ -4877,7 +4883,6 @@
         return_null_without_error, METH_NOARGS},
     {"return_result_with_error",
         return_result_with_error, METH_NOARGS},
-#ifndef PYPY_VERSION
     {"PyTime_FromSeconds", test_pytime_fromseconds,  METH_VARARGS},
     {"PyTime_FromSecondsObject", test_pytime_fromsecondsobject,  METH_VARARGS},
     {"PyTime_AsSecondsDouble", test_pytime_assecondsdouble, METH_VARARGS},
@@ -4887,6 +4892,7 @@
 #endif
     {"PyTime_AsMilliseconds", test_PyTime_AsMilliseconds, METH_VARARGS},
     {"PyTime_AsMicroseconds", test_PyTime_AsMicroseconds, METH_VARARGS},
+#ifndef PYPY_VERSION
     {"get_recursion_depth", get_recursion_depth, METH_NOARGS},
     {"pymem_buffer_overflow", pymem_buffer_overflow, METH_NOARGS},
     {"pymem_api_misuse", pymem_api_misuse, METH_NOARGS},
@@ -4906,10 +4912,10 @@
     {"pyobject_fastcalldict", test_pyobject_fastcalldict, METH_VARARGS},
     {"pyobject_fastcallkeywords", test_pyobject_fastcallkeywords, METH_VARARGS},
     {"stack_pointer", stack_pointer, METH_NOARGS},
+#endif
 #ifdef W_STOPCODE
     {"W_STOPCODE", py_w_stopcode, METH_VARARGS},
 #endif
-#endif /* PYPY_VERSION */
     {"get_mapping_keys", get_mapping_keys, METH_O},
     {"get_mapping_values", get_mapping_values, METH_O},
     {"get_mapping_items", get_mapping_items, METH_O},
diff --git a/lib_pypy/_tkinter/tklib_build.py b/lib_pypy/_tkinter/tklib_build.py
--- a/lib_pypy/_tkinter/tklib_build.py
+++ b/lib_pypy/_tkinter/tklib_build.py
@@ -36,8 +36,11 @@
         for _ver in ['8.6', '8.5', '']:
             incdirs = []
             linklibs = ['tcl' + _ver, 'tk' + _ver]
-            if os.path.isfile(''.join(['/usr/lib/lib', linklibs[1], '.so'])):
-                found = True
+            for lib in ['/usr/lib/lib', '/usr/lib64/lib']: 
+                if os.path.isfile(''.join([lib, linklibs[1], '.so'])):
+                    found = True
+                    break
+            if found:
                 break
     if not found:
         sys.stderr.write("*** TCL libraries not found!  Falling back...\n")
diff --git a/lib_pypy/cffi/_pycparser/README b/lib_pypy/cffi/_pycparser/README
--- a/lib_pypy/cffi/_pycparser/README
+++ b/lib_pypy/cffi/_pycparser/README
@@ -10,3 +10,8 @@
                     ^^^^^^^^^^^^^^^
             yacctab='cffi._pycparser.yacctab',
                      ^^^^^^^^^^^^^^^
+
+Also, when updating the version of this in-place, you must regenerate the
+lextab.py and yacctab.py files. They will be regenerated on import if they
+are not found, so they should be removed, then regenrated, then the new
+versions committed.
diff --git a/lib_pypy/cffi/_pycparser/__init__.py b/lib_pypy/cffi/_pycparser/__init__.py
--- a/lib_pypy/cffi/_pycparser/__init__.py
+++ b/lib_pypy/cffi/_pycparser/__init__.py
@@ -4,12 +4,14 @@
 # This package file exports some convenience functions for
 # interacting with pycparser
 #
-# Copyright (C) 2008-2015, Eli Bendersky
+# Eli Bendersky [https://eli.thegreenplace.net/]
 # License: BSD
 #-----------------------------------------------------------------
 __all__ = ['c_lexer', 'c_parser', 'c_ast']
-__version__ = '2.14'
+__version__ = '2.19'
 
+import io
+from subprocess import check_output
 from .c_parser import CParser
 
 
@@ -27,7 +29,6 @@
         When successful, returns the preprocessed file's contents.
         Errors from cpp will be printed out.
     """
-    from subprocess import Popen, PIPE
     path_list = [cpp_path]
     if isinstance(cpp_args, list):
         path_list += cpp_args
@@ -38,11 +39,7 @@
     try:
         # Note the use of universal_newlines to treat all newlines
         # as \n for Python's purpose
-        #
-        pipe = Popen(   path_list,
-                        stdout=PIPE,
-                        universal_newlines=True)
-        text = pipe.communicate()[0]
+        text = check_output(path_list, universal_newlines=True)
     except OSError as e:
         raise RuntimeError("Unable to invoke 'cpp'.  " +
             'Make sure its path was passed correctly\n' +
@@ -85,7 +82,7 @@
     if use_cpp:
         text = preprocess_file(filename, cpp_path, cpp_args)
     else:
-        with open(filename, 'rU') as f:
+        with io.open(filename) as f:
             text = f.read()
 
     if parser is None:
diff --git a/lib_pypy/cffi/_pycparser/_ast_gen.py b/lib_pypy/cffi/_pycparser/_ast_gen.py
--- a/lib_pypy/cffi/_pycparser/_ast_gen.py
+++ b/lib_pypy/cffi/_pycparser/_ast_gen.py
@@ -7,7 +7,7 @@
 # The design of this module was inspired by astgen.py from the
 # Python 2.5 code-base.
 #
-# Copyright (C) 2008-2015, Eli Bendersky
+# Eli Bendersky [https://eli.thegreenplace.net/]
 # License: BSD
 #-----------------------------------------------------------------
 import pprint
@@ -63,6 +63,7 @@
         contents: a list of contents - attributes and child nodes
         See comment at the top of the configuration file for details.
     """
+
     def __init__(self, name, contents):
         self.name = name
         self.all_entries = []
@@ -84,6 +85,8 @@
     def generate_source(self):
         src = self._gen_init()
         src += '\n' + self._gen_children()
+        src += '\n' + self._gen_iter()
+
         src += '\n' + self._gen_attr_names()
         return src
 
@@ -131,6 +134,33 @@
 
         return src
 
+    def _gen_iter(self):
+        src = '    def __iter__(self):\n'
+
+        if self.all_entries:
+            for child in self.child:
+                src += (
+                    '        if self.%(child)s is not None:\n' +
+                    '            yield self.%(child)s\n') % (dict(child=child))
+
+            for seq_child in self.seq_child:
+                src += (
+                    '        for child in (self.%(child)s or []):\n'
+                    '            yield child\n') % (dict(child=seq_child))
+
+            if not (self.child or self.seq_child):
+                # Empty generator
+                src += (
+                    '        return\n' +
+                    '        yield\n')
+        else:
+            # Empty generator
+            src += (
+                '        return\n' +
+                '        yield\n')
+
+        return src
+
     def _gen_attr_names(self):
         src = "    attr_names = (" + ''.join("%r, " % nm for nm in self.attr) + ')'
         return src
@@ -150,7 +180,7 @@
 #
 # AST Node classes.
 #
-# Copyright (C) 2008-2015, Eli Bendersky
+# Eli Bendersky [https://eli.thegreenplace.net/]
 # License: BSD
 #-----------------------------------------------------------------
 
@@ -159,11 +189,38 @@
 _PROLOGUE_CODE = r'''
 import sys
 
+def _repr(obj):
+    """
+    Get the representation of an object, with dedicated pprint-like format for lists.
+    """
+    if isinstance(obj, list):
+        return '[' + (',\n '.join((_repr(e).replace('\n', '\n ') for e in obj))) + '\n]'
+    else:
+        return repr(obj) 
 
 class Node(object):
     __slots__ = ()
     """ Abstract base class for AST nodes.
     """
+    def __repr__(self):
+        """ Generates a python representation of the current node
+        """
+        result = self.__class__.__name__ + '('
+        
+        indent = ''
+        separator = ''
+        for name in self.__slots__[:-2]:
+            result += separator
+            result += indent
+            result += name + '=' + (_repr(getattr(self, name)).replace('\n', '\n  ' + (' ' * (len(name) + len(self.__class__.__name__)))))
+            
+            separator = ','
+            indent = '\n ' + (' ' * len(self.__class__.__name__))
+        
+        result += indent + ')'
+        
+        return result
+
     def children(self):
         """ A sequence of all children that are Nodes
         """
@@ -253,26 +310,29 @@
         *   Modeled after Python's own AST visiting facilities
             (the ast module of Python 3.0)
     """
+
+    _method_cache = None
+
     def visit(self, node):
         """ Visit a node.
         """
-        method = 'visit_' + node.__class__.__name__
-        visitor = getattr(self, method, self.generic_visit)
+
+        if self._method_cache is None:
+            self._method_cache = {}
+
+        visitor = self._method_cache.get(node.__class__.__name__, None)
+        if visitor is None:
+            method = 'visit_' + node.__class__.__name__
+            visitor = getattr(self, method, self.generic_visit)
+            self._method_cache[node.__class__.__name__] = visitor
+
         return visitor(node)
 
     def generic_visit(self, node):
         """ Called if no explicit visitor function exists for a
             node. Implements preorder visiting of the node.
         """
-        for c_name, c in node.children():
+        for c in node:
             self.visit(c)
 
-
 '''
-
-
-if __name__ == "__main__":
-    import sys
-    ast_gen = ASTCodeGenerator('_c_ast.cfg')
-    ast_gen.generate(open('c_ast.py', 'w'))
-
diff --git a/lib_pypy/cffi/_pycparser/_build_tables.py b/lib_pypy/cffi/_pycparser/_build_tables.py
--- a/lib_pypy/cffi/_pycparser/_build_tables.py
+++ b/lib_pypy/cffi/_pycparser/_build_tables.py
@@ -6,17 +6,21 @@
 # Also generates AST code from the configuration file.
 # Should be called from the pycparser directory.
 #
-# Copyright (C) 2008-2015, Eli Bendersky
+# Eli Bendersky [https://eli.thegreenplace.net/]
 # License: BSD
 #-----------------------------------------------------------------
 
+# Insert '.' and '..' as first entries to the search path for modules.
+# Restricted environments like embeddable python do not include the
+# current working directory on startup.
+import sys
+sys.path[0:0] = ['.', '..']
+
 # Generate c_ast.py
 from _ast_gen import ASTCodeGenerator
 ast_gen = ASTCodeGenerator('_c_ast.cfg')
 ast_gen.generate(open('c_ast.py', 'w'))
 
-import sys
-sys.path[0:0] = ['.', '..']
 from pycparser import c_parser
 
 # Generates the tables
diff --git a/lib_pypy/cffi/_pycparser/_c_ast.cfg b/lib_pypy/cffi/_pycparser/_c_ast.cfg
--- a/lib_pypy/cffi/_pycparser/_c_ast.cfg
+++ b/lib_pypy/cffi/_pycparser/_c_ast.cfg
@@ -9,7 +9,7 @@
 #   <name>**    - a sequence of child nodes
 #   <name>      - an attribute
 #
-# Copyright (C) 2008-2015, Eli Bendersky
+# Eli Bendersky [https://eli.thegreenplace.net/]
 # License: BSD
 #-----------------------------------------------------------------
 
@@ -187,3 +187,5 @@
 Union: [name, decls**]
 
 While: [cond*, stmt*]
+
+Pragma: [string]
diff --git a/lib_pypy/cffi/_pycparser/ast_transforms.py b/lib_pypy/cffi/_pycparser/ast_transforms.py
--- a/lib_pypy/cffi/_pycparser/ast_transforms.py
+++ b/lib_pypy/cffi/_pycparser/ast_transforms.py
@@ -3,7 +3,7 @@
 #
 # Some utilities used by the parser to create a friendlier AST.
 #
-# Copyright (C) 2008-2015, Eli Bendersky
+# Eli Bendersky [https://eli.thegreenplace.net/]
 # License: BSD
 #------------------------------------------------------------------------------
 
@@ -43,7 +43,7 @@
                     Default:
                         break
 
-        The goal of this transform it to fix this mess, turning it into the
+        The goal of this transform is to fix this mess, turning it into the
         following:
 
             Switch
@@ -74,7 +74,8 @@
 
     # Goes over the children of the Compound below the Switch, adding them
     # either directly below new_compound or below the last Case as appropriate
-    for child in switch_node.stmt.block_items:
+    # (for `switch(cond) {}`, block_items would have been None)
+    for child in (switch_node.stmt.block_items or []):
         if isinstance(child, (c_ast.Case, c_ast.Default)):
             # If it's a Case/Default:
             # 1. Add it to the Compound and mark as "last case"
diff --git a/lib_pypy/cffi/_pycparser/c_ast.py b/lib_pypy/cffi/_pycparser/c_ast.py
--- a/lib_pypy/cffi/_pycparser/c_ast.py
+++ b/lib_pypy/cffi/_pycparser/c_ast.py
@@ -11,18 +11,45 @@
 #
 # AST Node classes.
 #
-# Copyright (C) 2008-2015, Eli Bendersky
+# Eli Bendersky [https://eli.thegreenplace.net/]
 # License: BSD
 #-----------------------------------------------------------------
 
 
 import sys
 
+def _repr(obj):
+    """
+    Get the representation of an object, with dedicated pprint-like format for lists.
+    """
+    if isinstance(obj, list):
+        return '[' + (',\n '.join((_repr(e).replace('\n', '\n ') for e in obj))) + '\n]'
+    else:
+        return repr(obj) 
 
 class Node(object):
     __slots__ = ()
     """ Abstract base class for AST nodes.
     """
+    def __repr__(self):
+        """ Generates a python representation of the current node
+        """
+        result = self.__class__.__name__ + '('
+        
+        indent = ''
+        separator = ''
+        for name in self.__slots__[:-2]:
+            result += separator
+            result += indent
+            result += name + '=' + (_repr(getattr(self, name)).replace('\n', '\n  ' + (' ' * (len(name) + len(self.__class__.__name__)))))
+            
+            separator = ','
+            indent = '\n ' + (' ' * len(self.__class__.__name__))
+        
+        result += indent + ')'
+        
+        return result
+
     def children(self):
         """ A sequence of all children that are Nodes
         """
@@ -112,21 +139,31 @@
         *   Modeled after Python's own AST visiting facilities
             (the ast module of Python 3.0)
     """
+
+    _method_cache = None
+
     def visit(self, node):
         """ Visit a node.
         """
-        method = 'visit_' + node.__class__.__name__
-        visitor = getattr(self, method, self.generic_visit)
+
+        if self._method_cache is None:
+            self._method_cache = {}
+
+        visitor = self._method_cache.get(node.__class__.__name__, None)
+        if visitor is None:
+            method = 'visit_' + node.__class__.__name__
+            visitor = getattr(self, method, self.generic_visit)
+            self._method_cache[node.__class__.__name__] = visitor
+
         return visitor(node)
 
     def generic_visit(self, node):
         """ Called if no explicit visitor function exists for a
             node. Implements preorder visiting of the node.
         """
-        for c_name, c in node.children():
+        for c in node:
             self.visit(c)
 
-
 class ArrayDecl(Node):
     __slots__ = ('type', 'dim', 'dim_quals', 'coord', '__weakref__')
     def __init__(self, type, dim, dim_quals, coord=None):
@@ -141,6 +178,12 @@
         if self.dim is not None: nodelist.append(("dim", self.dim))
         return tuple(nodelist)
 
+    def __iter__(self):
+        if self.type is not None:
+            yield self.type
+        if self.dim is not None:
+            yield self.dim
+
     attr_names = ('dim_quals', )
 
 class ArrayRef(Node):
@@ -156,6 +199,12 @@
         if self.subscript is not None: nodelist.append(("subscript", self.subscript))
         return tuple(nodelist)
 
+    def __iter__(self):
+        if self.name is not None:
+            yield self.name
+        if self.subscript is not None:
+            yield self.subscript
+
     attr_names = ()
 
 class Assignment(Node):
@@ -172,6 +221,12 @@
         if self.rvalue is not None: nodelist.append(("rvalue", self.rvalue))
         return tuple(nodelist)
 
+    def __iter__(self):
+        if self.lvalue is not None:
+            yield self.lvalue
+        if self.rvalue is not None:
+            yield self.rvalue
+
     attr_names = ('op', )
 
 class BinaryOp(Node):
@@ -188,6 +243,12 @@
         if self.right is not None: nodelist.append(("right", self.right))
         return tuple(nodelist)
 
+    def __iter__(self):
+        if self.left is not None:
+            yield self.left
+        if self.right is not None:
+            yield self.right
+
     attr_names = ('op', )
 
 class Break(Node):
@@ -198,6 +259,10 @@
     def children(self):
         return ()
 
+    def __iter__(self):
+        return
+        yield
+
     attr_names = ()
 
 class Case(Node):
@@ -214,6 +279,12 @@
             nodelist.append(("stmts[%d]" % i, child))
         return tuple(nodelist)
 
+    def __iter__(self):
+        if self.expr is not None:
+            yield self.expr
+        for child in (self.stmts or []):
+            yield child
+
     attr_names = ()
 
 class Cast(Node):
@@ -229,6 +300,12 @@
         if self.expr is not None: nodelist.append(("expr", self.expr))
         return tuple(nodelist)
 
+    def __iter__(self):
+        if self.to_type is not None:
+            yield self.to_type
+        if self.expr is not None:
+            yield self.expr
+
     attr_names = ()
 
 class Compound(Node):
@@ -243,6 +320,10 @@
             nodelist.append(("block_items[%d]" % i, child))
         return tuple(nodelist)
 
+    def __iter__(self):
+        for child in (self.block_items or []):
+            yield child
+
     attr_names = ()
 
 class CompoundLiteral(Node):
@@ -258,6 +339,12 @@
         if self.init is not None: nodelist.append(("init", self.init))
         return tuple(nodelist)
 
+    def __iter__(self):
+        if self.type is not None:
+            yield self.type
+        if self.init is not None:
+            yield self.init
+
     attr_names = ()
 
 class Constant(Node):
@@ -271,6 +358,10 @@
         nodelist = []
         return tuple(nodelist)
 
+    def __iter__(self):
+        return
+        yield
+
     attr_names = ('type', 'value', )
 
 class Continue(Node):
@@ -281,6 +372,10 @@
     def children(self):
         return ()
 
+    def __iter__(self):
+        return
+        yield
+
     attr_names = ()
 
 class Decl(Node):
@@ -302,6 +397,14 @@
         if self.bitsize is not None: nodelist.append(("bitsize", self.bitsize))
         return tuple(nodelist)
 
+    def __iter__(self):
+        if self.type is not None:
+            yield self.type
+        if self.init is not None:
+            yield self.init
+        if self.bitsize is not None:
+            yield self.bitsize
+
     attr_names = ('name', 'quals', 'storage', 'funcspec', )
 
 class DeclList(Node):
@@ -316,6 +419,10 @@
             nodelist.append(("decls[%d]" % i, child))
         return tuple(nodelist)
 
+    def __iter__(self):
+        for child in (self.decls or []):
+            yield child
+
     attr_names = ()
 
 class Default(Node):
@@ -330,6 +437,10 @@
             nodelist.append(("stmts[%d]" % i, child))
         return tuple(nodelist)
 
+    def __iter__(self):
+        for child in (self.stmts or []):
+            yield child
+
     attr_names = ()
 
 class DoWhile(Node):
@@ -345,6 +456,12 @@
         if self.stmt is not None: nodelist.append(("stmt", self.stmt))
         return tuple(nodelist)
 
+    def __iter__(self):
+        if self.cond is not None:
+            yield self.cond
+        if self.stmt is not None:
+            yield self.stmt
+
     attr_names = ()
 
 class EllipsisParam(Node):
@@ -355,6 +472,10 @@
     def children(self):
         return ()
 
+    def __iter__(self):
+        return
+        yield
+
     attr_names = ()
 
 class EmptyStatement(Node):
@@ -365,6 +486,10 @@
     def children(self):
         return ()
 
+    def __iter__(self):
+        return
+        yield
+
     attr_names = ()
 
 class Enum(Node):
@@ -379,6 +504,10 @@
         if self.values is not None: nodelist.append(("values", self.values))
         return tuple(nodelist)
 
+    def __iter__(self):
+        if self.values is not None:
+            yield self.values
+
     attr_names = ('name', )
 
 class Enumerator(Node):
@@ -393,6 +522,10 @@
         if self.value is not None: nodelist.append(("value", self.value))
         return tuple(nodelist)
 
+    def __iter__(self):
+        if self.value is not None:
+            yield self.value
+
     attr_names = ('name', )
 
 class EnumeratorList(Node):
@@ -407,6 +540,10 @@
             nodelist.append(("enumerators[%d]" % i, child))
         return tuple(nodelist)
 
+    def __iter__(self):
+        for child in (self.enumerators or []):
+            yield child
+
     attr_names = ()
 
 class ExprList(Node):
@@ -421,6 +558,10 @@
             nodelist.append(("exprs[%d]" % i, child))
         return tuple(nodelist)
 
+    def __iter__(self):
+        for child in (self.exprs or []):
+            yield child
+
     attr_names = ()
 
 class FileAST(Node):
@@ -435,6 +576,10 @@
             nodelist.append(("ext[%d]" % i, child))
         return tuple(nodelist)
 
+    def __iter__(self):
+        for child in (self.ext or []):
+            yield child
+
     attr_names = ()
 
 class For(Node):
@@ -454,6 +599,16 @@
         if self.stmt is not None: nodelist.append(("stmt", self.stmt))
         return tuple(nodelist)
 
+    def __iter__(self):
+        if self.init is not None:
+            yield self.init
+        if self.cond is not None:
+            yield self.cond
+        if self.next is not None:
+            yield self.next
+        if self.stmt is not None:
+            yield self.stmt
+
     attr_names = ()
 
 class FuncCall(Node):
@@ -469,6 +624,12 @@
         if self.args is not None: nodelist.append(("args", self.args))
         return tuple(nodelist)
 
+    def __iter__(self):
+        if self.name is not None:
+            yield self.name
+        if self.args is not None:
+            yield self.args
+
     attr_names = ()
 
 class FuncDecl(Node):
@@ -484,6 +645,12 @@
         if self.type is not None: nodelist.append(("type", self.type))
         return tuple(nodelist)
 
+    def __iter__(self):
+        if self.args is not None:
+            yield self.args
+        if self.type is not None:
+            yield self.type
+
     attr_names = ()
 
 class FuncDef(Node):
@@ -502,6 +669,14 @@
             nodelist.append(("param_decls[%d]" % i, child))
         return tuple(nodelist)
 
+    def __iter__(self):
+        if self.decl is not None:
+            yield self.decl
+        if self.body is not None:
+            yield self.body
+        for child in (self.param_decls or []):
+            yield child
+
     attr_names = ()
 
 class Goto(Node):
@@ -514,6 +689,10 @@
         nodelist = []
         return tuple(nodelist)
 
+    def __iter__(self):
+        return
+        yield
+
     attr_names = ('name', )
 
 class ID(Node):
@@ -526,6 +705,10 @@
         nodelist = []
         return tuple(nodelist)
 
+    def __iter__(self):
+        return
+        yield
+
     attr_names = ('name', )
 
 class IdentifierType(Node):
@@ -538,6 +721,10 @@
         nodelist = []
         return tuple(nodelist)
 
+    def __iter__(self):
+        return
+        yield
+
     attr_names = ('names', )
 
 class If(Node):
@@ -555,6 +742,14 @@
         if self.iffalse is not None: nodelist.append(("iffalse", self.iffalse))
         return tuple(nodelist)
 
+    def __iter__(self):
+        if self.cond is not None:
+            yield self.cond
+        if self.iftrue is not None:
+            yield self.iftrue
+        if self.iffalse is not None:
+            yield self.iffalse
+
     attr_names = ()
 
 class InitList(Node):
@@ -569,6 +764,10 @@
             nodelist.append(("exprs[%d]" % i, child))
         return tuple(nodelist)
 
+    def __iter__(self):
+        for child in (self.exprs or []):
+            yield child
+
     attr_names = ()
 
 class Label(Node):
@@ -583,6 +782,10 @@
         if self.stmt is not None: nodelist.append(("stmt", self.stmt))
         return tuple(nodelist)
 
+    def __iter__(self):
+        if self.stmt is not None:
+            yield self.stmt
+
     attr_names = ('name', )
 
 class NamedInitializer(Node):
@@ -599,6 +802,12 @@
             nodelist.append(("name[%d]" % i, child))
         return tuple(nodelist)
 
+    def __iter__(self):
+        if self.expr is not None:
+            yield self.expr
+        for child in (self.name or []):
+            yield child
+
     attr_names = ()
 
 class ParamList(Node):
@@ -613,6 +822,10 @@
             nodelist.append(("params[%d]" % i, child))
         return tuple(nodelist)
 
+    def __iter__(self):
+        for child in (self.params or []):
+            yield child
+
     attr_names = ()
 
 class PtrDecl(Node):
@@ -627,6 +840,10 @@
         if self.type is not None: nodelist.append(("type", self.type))
         return tuple(nodelist)
 
+    def __iter__(self):
+        if self.type is not None:
+            yield self.type
+
     attr_names = ('quals', )
 
 class Return(Node):
@@ -640,6 +857,10 @@
         if self.expr is not None: nodelist.append(("expr", self.expr))
         return tuple(nodelist)
 
+    def __iter__(self):
+        if self.expr is not None:
+            yield self.expr
+
     attr_names = ()
 
 class Struct(Node):
@@ -655,6 +876,10 @@
             nodelist.append(("decls[%d]" % i, child))
         return tuple(nodelist)
 
+    def __iter__(self):
+        for child in (self.decls or []):
+            yield child
+
     attr_names = ('name', )
 
 class StructRef(Node):
@@ -671,6 +896,12 @@
         if self.field is not None: nodelist.append(("field", self.field))
         return tuple(nodelist)
 
+    def __iter__(self):
+        if self.name is not None:
+            yield self.name
+        if self.field is not None:
+            yield self.field
+
     attr_names = ('type', )
 
 class Switch(Node):
@@ -686,6 +917,12 @@
         if self.stmt is not None: nodelist.append(("stmt", self.stmt))
         return tuple(nodelist)
 
+    def __iter__(self):
+        if self.cond is not None:
+            yield self.cond
+        if self.stmt is not None:
+            yield self.stmt
+
     attr_names = ()
 
 class TernaryOp(Node):
@@ -703,6 +940,14 @@
         if self.iffalse is not None: nodelist.append(("iffalse", self.iffalse))
         return tuple(nodelist)
 
+    def __iter__(self):
+        if self.cond is not None:
+            yield self.cond
+        if self.iftrue is not None:
+            yield self.iftrue
+        if self.iffalse is not None:
+            yield self.iffalse
+
     attr_names = ()
 
 class TypeDecl(Node):
@@ -718,6 +963,10 @@
         if self.type is not None: nodelist.append(("type", self.type))
         return tuple(nodelist)
 
+    def __iter__(self):
+        if self.type is not None:
+            yield self.type
+
     attr_names = ('declname', 'quals', )
 
 class Typedef(Node):
@@ -734,6 +983,10 @@
         if self.type is not None: nodelist.append(("type", self.type))
         return tuple(nodelist)
 
+    def __iter__(self):
+        if self.type is not None:
+            yield self.type
+
     attr_names = ('name', 'quals', 'storage', )
 
 class Typename(Node):
@@ -749,6 +1002,10 @@
         if self.type is not None: nodelist.append(("type", self.type))
         return tuple(nodelist)
 
+    def __iter__(self):
+        if self.type is not None:
+            yield self.type
+
     attr_names = ('name', 'quals', )
 
 class UnaryOp(Node):
@@ -763,6 +1020,10 @@
         if self.expr is not None: nodelist.append(("expr", self.expr))
         return tuple(nodelist)
 
+    def __iter__(self):
+        if self.expr is not None:
+            yield self.expr
+
     attr_names = ('op', )
 
 class Union(Node):
@@ -778,6 +1039,10 @@
             nodelist.append(("decls[%d]" % i, child))
         return tuple(nodelist)
 
+    def __iter__(self):
+        for child in (self.decls or []):
+            yield child
+
     attr_names = ('name', )
 
 class While(Node):
@@ -793,5 +1058,27 @@
         if self.stmt is not None: nodelist.append(("stmt", self.stmt))
         return tuple(nodelist)
 
+    def __iter__(self):
+        if self.cond is not None:
+            yield self.cond
+        if self.stmt is not None:
+            yield self.stmt
+
     attr_names = ()
 
+class Pragma(Node):
+    __slots__ = ('string', 'coord', '__weakref__')
+    def __init__(self, string, coord=None):
+        self.string = string
+        self.coord = coord
+
+    def children(self):
+        nodelist = []
+        return tuple(nodelist)
+
+    def __iter__(self):
+        return
+        yield
+
+    attr_names = ('string', )
+
diff --git a/lib_pypy/cffi/_pycparser/c_generator.py b/lib_pypy/cffi/_pycparser/c_generator.py
--- a/lib_pypy/cffi/_pycparser/c_generator.py
+++ b/lib_pypy/cffi/_pycparser/c_generator.py
@@ -3,7 +3,7 @@
 #
 # C code generator from pycparser AST nodes.
 #
-# Copyright (C) 2008-2015, Eli Bendersky
+# Eli Bendersky [https://eli.thegreenplace.net/]
 # License: BSD
 #------------------------------------------------------------------------------
 from . import c_ast
@@ -40,6 +40,12 @@
     def visit_ID(self, n):
         return n.name
 
+    def visit_Pragma(self, n):
+        ret = '#pragma'
+        if n.string:
+            ret += ' ' + n.string
+        return ret
+
     def visit_ArrayRef(self, n):
         arrref = self._parenthesize_unless_simple(n.name)
         return arrref + '[' + self.visit(n.subscript) + ']'
@@ -113,7 +119,7 @@
         return s
 
     def visit_Cast(self, n):
-        s = '(' + self._generate_type(n.to_type) + ')'
+        s = '(' + self._generate_type(n.to_type, emit_declname=False) + ')'
         return s + ' ' + self._parenthesize_unless_simple(n.expr)
 
     def visit_ExprList(self, n):
@@ -129,18 +135,20 @@
         return ', '.join(visited_subexprs)
 
     def visit_Enum(self, n):
-        s = 'enum'
-        if n.name: s += ' ' + n.name
-        if n.values:
-            s += ' {'
-            for i, enumerator in enumerate(n.values.enumerators):
-                s += enumerator.name
-                if enumerator.value:
-                    s += ' = ' + self.visit(enumerator.value)
-                if i != len(n.values.enumerators) - 1:
-                    s += ', '
-            s += '}'
-        return s
+        return self._generate_struct_union_enum(n, name='enum')
+
+    def visit_Enumerator(self, n):
+        if not n.value:
+            return '{indent}{name},\n'.format(
+                indent=self._make_indent(),
+                name=n.name,
+            )
+        else:
+            return '{indent}{name} = {value},\n'.format(
+                indent=self._make_indent(),
+                name=n.name,
+                value=self.visit(n.value),
+            )
 
     def visit_FuncDef(self, n):
         decl = self.visit(n.decl)
@@ -157,6 +165,8 @@
         for ext in n.ext:
             if isinstance(ext, c_ast.FuncDef):
                 s += self.visit(ext)
+            elif isinstance(ext, c_ast.Pragma):
+                s += self.visit(ext) + '\n'
             else:
                 s += self.visit(ext) + ';\n'
         return s
@@ -170,6 +180,10 @@
         s += self._make_indent() + '}\n'
         return s
 
+    def visit_CompoundLiteral(self, n):
+        return '(' + self.visit(n.type) + '){' + self.visit(n.init) + '}'
+
+
     def visit_EmptyStatement(self, n):
         return ';'
 
@@ -188,9 +202,9 @@
         return 'continue;'
 
     def visit_TernaryOp(self, n):
-        s = self._visit_expr(n.cond) + ' ? '
-        s += self._visit_expr(n.iftrue) + ' : '
-        s += self._visit_expr(n.iffalse)
+        s  = '(' + self._visit_expr(n.cond) + ') ? '
+        s += '(' + self._visit_expr(n.iftrue) + ') : '
+        s += '(' + self._visit_expr(n.iffalse) + ')'
         return s
 
     def visit_If(self, n):
@@ -256,43 +270,67 @@
         return '...'
 
     def visit_Struct(self, n):
-        return self._generate_struct_union(n, 'struct')
+        return self._generate_struct_union_enum(n, 'struct')
 
     def visit_Typename(self, n):
         return self._generate_type(n.type)
 
     def visit_Union(self, n):
-        return self._generate_struct_union(n, 'union')
+        return self._generate_struct_union_enum(n, 'union')
 
     def visit_NamedInitializer(self, n):
         s = ''
         for name in n.name:
             if isinstance(name, c_ast.ID):
                 s += '.' + name.name
-            elif isinstance(name, c_ast.Constant):
-                s += '[' + name.value + ']'
-        s += ' = ' + self.visit(n.expr)
+            else:
+                s += '[' + self.visit(name) + ']'
+        s += ' = ' + self._visit_expr(n.expr)
         return s
 
     def visit_FuncDecl(self, n):
         return self._generate_type(n)
 
-    def _generate_struct_union(self, n, name):
-        """ Generates code for structs and unions. name should be either
-            'struct' or union.
+    def visit_ArrayDecl(self, n):
+        return self._generate_type(n, emit_declname=False)
+
+    def visit_TypeDecl(self, n):
+        return self._generate_type(n, emit_declname=False)
+
+    def visit_PtrDecl(self, n):
+        return self._generate_type(n, emit_declname=False)
+
+    def _generate_struct_union_enum(self, n, name):
+        """ Generates code for structs, unions, and enums. name should be
+            'struct', 'union', or 'enum'.
         """
+        if name in ('struct', 'union'):
+            members = n.decls
+            body_function = self._generate_struct_union_body
+        else:
+            assert name == 'enum'
+            members = None if n.values is None else n.values.enumerators
+            body_function = self._generate_enum_body
         s = name + ' ' + (n.name or '')
-        if n.decls:
+        if members is not None:
+            # None means no members
+            # Empty sequence means an empty list of members
             s += '\n'
             s += self._make_indent()
             self.indent_level += 2
             s += '{\n'
-            for decl in n.decls:
-                s += self._generate_stmt(decl)
+            s += body_function(members)
             self.indent_level -= 2
             s += self._make_indent() + '}'
         return s
 
+    def _generate_struct_union_body(self, members):
+        return ''.join(self._generate_stmt(decl) for decl in members)
+
+    def _generate_enum_body(self, members):
+        # `[:-2] + '\n'` removes the final `,` from the enumerator list
+        return ''.join(self.visit(value) for value in members)[:-2] + '\n'
+
     def _generate_stmt(self, n, add_indent=False):
         """ Generation from a statement node. This method exists as a wrapper
             for individual visit_* methods to handle different treatment of
@@ -330,7 +368,7 @@
         s += self._generate_type(n.type)
         return s
 
-    def _generate_type(self, n, modifiers=[]):
+    def _generate_type(self, n, modifiers=[], emit_declname = True):
         """ Recursive generation from a type node. n is the type node.
             modifiers collects the PtrDecl, ArrayDecl and FuncDecl modifiers
             encountered on the way down to a TypeDecl, to allow proper
@@ -344,23 +382,29 @@
             if n.quals: s += ' '.join(n.quals) + ' '
             s += self.visit(n.type)
 
-            nstr = n.declname if n.declname else ''
+            nstr = n.declname if n.declname and emit_declname else ''
             # Resolve modifiers.
             # Wrap in parens to distinguish pointer to array and pointer to
             # function syntax.
             #
             for i, modifier in enumerate(modifiers):
                 if isinstance(modifier, c_ast.ArrayDecl):
-                    if (i != 0 and isinstance(modifiers[i - 1], c_ast.PtrDecl)):
-                        nstr = '(' + nstr + ')'
-                    nstr += '[' + self.visit(modifier.dim) + ']'
+                    if (i != 0 and
+                        isinstance(modifiers[i - 1], c_ast.PtrDecl)):
+                            nstr = '(' + nstr + ')'
+                    nstr += '['
+                    if modifier.dim_quals:
+                        nstr += ' '.join(modifier.dim_quals) + ' '
+                    nstr += self.visit(modifier.dim) + ']'
                 elif isinstance(modifier, c_ast.FuncDecl):
-                    if (i != 0 and isinstance(modifiers[i - 1], c_ast.PtrDecl)):
-                        nstr = '(' + nstr + ')'
+                    if (i != 0 and
+                        isinstance(modifiers[i - 1], c_ast.PtrDecl)):
+                            nstr = '(' + nstr + ')'
                     nstr += '(' + self.visit(modifier.args) + ')'
                 elif isinstance(modifier, c_ast.PtrDecl):
                     if modifier.quals:
-                        nstr = '* %s %s' % (' '.join(modifier.quals), nstr)
+                        nstr = '* %s%s' % (' '.join(modifier.quals),
+                                           ' ' + nstr if nstr else '')
                     else:
                         nstr = '*' + nstr
             if nstr: s += ' ' + nstr
@@ -368,11 +412,12 @@
         elif typ == c_ast.Decl:
             return self._generate_decl(n.type)
         elif typ == c_ast.Typename:
-            return self._generate_type(n.type)
+            return self._generate_type(n.type, emit_declname = emit_declname)
         elif typ == c_ast.IdentifierType:
             return ' '.join(n.names) + ' '
         elif typ in (c_ast.ArrayDecl, c_ast.PtrDecl, c_ast.FuncDecl):
-            return self._generate_type(n.type, modifiers + [n])
+            return self._generate_type(n.type, modifiers + [n],
+                                       emit_declname = emit_declname)
         else:
             return self.visit(n)
 
@@ -395,5 +440,5 @@
         """ Returns True for nodes that are "simple" - i.e. nodes that always
             have higher precedence than operators.
         """
-        return isinstance(n,(   c_ast.Constant, c_ast.ID, c_ast.ArrayRef,
-                                c_ast.StructRef, c_ast.FuncCall))
+        return isinstance(n, (c_ast.Constant, c_ast.ID, c_ast.ArrayRef,
+                              c_ast.StructRef, c_ast.FuncCall))
diff --git a/lib_pypy/cffi/_pycparser/c_lexer.py b/lib_pypy/cffi/_pycparser/c_lexer.py
--- a/lib_pypy/cffi/_pycparser/c_lexer.py
+++ b/lib_pypy/cffi/_pycparser/c_lexer.py
@@ -3,7 +3,7 @@
 #
 # CLexer class: lexer for the C language
 #
-# Copyright (C) 2008-2015, Eli Bendersky
+# Eli Bendersky [https://eli.thegreenplace.net/]
 # License: BSD
 #------------------------------------------------------------------------------
 import re
@@ -19,7 +19,7 @@
         tokens.
 
         The public attribute filename can be set to an initial
-        filaneme, but the lexer will update it upon #line
+        filename, but the lexer will update it upon #line
         directives.
     """
     def __init__(self, error_func, on_lbrace_func, on_rbrace_func,
@@ -52,8 +52,8 @@
         # Allow either "# line" or "# <num>" to support GCC's
         # cpp output
         #
-        self.line_pattern = re.compile('([ \t]*line\W)|([ \t]*\d+)')
-        self.pragma_pattern = re.compile('[ \t]*pragma\W')
+        self.line_pattern = re.compile(r'([ \t]*line\W)|([ \t]*\d+)')
+        self.pragma_pattern = re.compile(r'[ \t]*pragma\W')
 
     def build(self, **kwargs):
         """ Builds the lexer from the specification. Must be
@@ -102,11 +102,11 @@
     keywords = (
         '_BOOL', '_COMPLEX', 'AUTO', 'BREAK', 'CASE', 'CHAR', 'CONST',
         'CONTINUE', 'DEFAULT', 'DO', 'DOUBLE', 'ELSE', 'ENUM', 'EXTERN',
-        'FLOAT', 'FOR', 'GOTO', 'IF', 'INLINE', 'INT', 'LONG', 
+        'FLOAT', 'FOR', 'GOTO', 'IF', 'INLINE', 'INT', 'LONG',
         'REGISTER', 'OFFSETOF',
         'RESTRICT', 'RETURN', 'SHORT', 'SIGNED', 'SIZEOF', 'STATIC', 'STRUCT',
         'SWITCH', 'TYPEDEF', 'UNION', 'UNSIGNED', 'VOID',
-        'VOLATILE', 'WHILE',
+        'VOLATILE', 'WHILE', '__INT128',
     )
 
     keyword_map = {}
@@ -171,7 +171,9 @@
         'ELLIPSIS',
 
         # pre-processor
-        'PPHASH',      # '#'
+        'PPHASH',       # '#'
+        'PPPRAGMA',     # 'pragma'
+        'PPPRAGMASTR',
     )
 
     ##
@@ -203,12 +205,37 @@
     # parse all correct code, even if it means to sometimes parse incorrect
     # code.
     #
-    simple_escape = r"""([a-zA-Z._~!=&\^\-\\?'"])"""
-    decimal_escape = r"""(\d+)"""
-    hex_escape = r"""(x[0-9a-fA-F]+)"""
-    bad_escape = r"""([\\][^a-zA-Z._~^!=&\^\-\\?'"x0-7])"""
+    # The original regexes were taken verbatim from the C syntax definition,
+    # and were later modified to avoid worst-case exponential running time.
+    #
+    #   simple_escape = r"""([a-zA-Z._~!=&\^\-\\?'"])"""
+    #   decimal_escape = r"""(\d+)"""
+    #   hex_escape = r"""(x[0-9a-fA-F]+)"""
+    #   bad_escape = r"""([\\][^a-zA-Z._~^!=&\^\-\\?'"x0-7])"""
+    #
+    # The following modifications were made to avoid the ambiguity that allowed backtracking:
+    # (https://github.com/eliben/pycparser/issues/61)
+    #
+    # - \x was removed from simple_escape, unless it was not followed by a hex digit, to avoid ambiguity with hex_escape.
+    # - hex_escape allows one or more hex characters, but requires that the next character(if any) is not hex
+    # - decimal_escape allows one or more decimal characters, but requires that the next character(if any) is not a decimal
+    # - bad_escape does not allow any decimals (8-9), to avoid conflicting with the permissive decimal_escape.
+    #
+    # Without this change, python's `re` module would recursively try parsing each ambiguous escape sequence in multiple ways.
+    # e.g. `\123` could be parsed as `\1`+`23`, `\12`+`3`, and `\123`.
+
+    simple_escape = r"""([a-wyzA-Z._~!=&\^\-\\?'"]|x(?![0-9a-fA-F]))"""
+    decimal_escape = r"""(\d+)(?!\d)"""
+    hex_escape = r"""(x[0-9a-fA-F]+)(?![0-9a-fA-F])"""
+    bad_escape = r"""([\\][^a-zA-Z._~^!=&\^\-\\?'"x0-9])"""
 
     escape_sequence = r"""(\\("""+simple_escape+'|'+decimal_escape+'|'+hex_escape+'))'
+
+    # This complicated regex with lookahead might be slow for strings, so because all of the valid escapes (including \x) allowed
+    # 0 or more non-escaped characters after the first character, simple_escape+decimal_escape+hex_escape got simplified to
+
+    escape_sequence_start_in_string = r"""(\\[0-9a-zA-Z._~!=&\^\-\\?'"])"""
+
     cconst_char = r"""([^'\\\n]|"""+escape_sequence+')'
     char_const = "'"+cconst_char+"'"
     wchar_const = 'L'+char_const
@@ -216,7 +243,7 @@
     bad_char_const = r"""('"""+cconst_char+"""[^'\n]+')|('')|('"""+bad_escape+r"""[^'\n]*')"""
 
     # string literals (K&R2: A.2.6)
-    string_char = r"""([^"\\\n]|"""+escape_sequence+')'
+    string_char = r"""([^"\\\n]|"""+escape_sequence_start_in_string+')'
     string_literal = '"'+string_char+'*"'
     wstring_literal = 'L'+string_literal
     bad_string_literal = '"'+string_char+'*'+bad_escape+string_char+'*"'
@@ -274,7 +301,6 @@
 
     def t_ppline_NEWLINE(self, t):
         r'\n'
-
         if self.pp_line is None:
             self._error('line number missing in #line', t)
         else:
@@ -304,15 +330,14 @@
 
     def t_pppragma_PPPRAGMA(self, t):
         r'pragma'
-        pass
+        return t
 
-    t_pppragma_ignore = ' \t<>.-{}();=+-*/$%@&^~!?:,0123456789'
+    t_pppragma_ignore = ' \t'
 
-    @TOKEN(string_literal)
-    def t_pppragma_STR(self, t): pass
-
-    @TOKEN(identifier)
-    def t_pppragma_ID(self, t): pass
+    def t_pppragma_STR(self, t):
+        '.+'
+        t.type = 'PPPRAGMASTR'
+        return t
 
     def t_pppragma_error(self, t):
         self._error('invalid #pragma directive', t)
@@ -482,4 +507,3 @@
     def t_error(self, t):
         msg = 'Illegal character %s' % repr(t.value[0])
         self._error(msg, t)
-
diff --git a/lib_pypy/cffi/_pycparser/c_parser.py b/lib_pypy/cffi/_pycparser/c_parser.py
--- a/lib_pypy/cffi/_pycparser/c_parser.py
+++ b/lib_pypy/cffi/_pycparser/c_parser.py
@@ -3,7 +3,7 @@
 #
 # CParser class: Parser and AST builder for the C language
 #
-# Copyright (C) 2008-2015, Eli Bendersky
+# Eli Bendersky [https://eli.thegreenplace.net/]
 # License: BSD
 #------------------------------------------------------------------------------
 import re
@@ -12,14 +12,16 @@
 
 from . import c_ast
 from .c_lexer import CLexer
-from .plyparser import PLYParser, Coord, ParseError
+from .plyparser import PLYParser, Coord, ParseError, parameterized, template
 from .ast_transforms import fix_switch_cases
 
 
+ at template
 class CParser(PLYParser):
     def __init__(
             self,
             lex_optimize=True,
+            lexer=CLexer,
             lextab='cffi._pycparser.lextab',
             yacc_optimize=True,
             yacctab='cffi._pycparser.yacctab',
@@ -42,6 +44,10 @@
                 to save the re-generation of the lexer table on
                 each run.
 
+            lexer:
+                Set this parameter to define the lexer to use if
+                you're not using the default CLexer.
+
             lextab:
                 Points to the lex table that's used for optimized
                 mode. Only if you're modifying the lexer and want
@@ -70,7 +76,7 @@
                 Set this parameter to control the location of generated
                 lextab and yacctab files.
         """
-        self.clex = CLexer(
+        self.clex = lexer(
             error_func=self._lex_error_func,
             on_lbrace_func=self._lex_on_lbrace_func,

From pypy.commits at gmail.com  Tue Oct  1 16:53:28 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 01 Oct 2019 13:53:28 -0700 (PDT)
Subject: [pypy-commit] pypy default: tag, update repackage script
Message-ID: <5d93bcc8.1c69fb81.72847.8653@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r97704:18a5bcf36924
Date: 2019-10-01 23:25 +0300
http://bitbucket.org/pypy/pypy/changeset/18a5bcf36924/

Log:	tag, update repackage script

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -50,4 +50,5 @@
 de061d87e39c7df4e436974096d7982c676a859d release-pypy3.6-v7.1.0
 784b254d669919c872a505b807db8462b6140973 release-pypy3.6-v7.1.1
 8cdda8b8cdb8ff29d9e620cccd6c5edd2f2a23ec release-pypy2.7-v7.1.1
-
+85dae4fd5c234b482feff834c73e089872194541 release-pypy2.7-v7.2.0rc0
+7ffb92269488f37c707ce66076f50ffd8613f8e2 release-pypy3.6-v7.2.0rc0
diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,15 +1,14 @@
 # Edit these appropriately before running this script
 pmaj=2  # python main version: 2 or 3
 pmin=7  # python minor version
-exe=pypy3 # pypy3 or pypy
+exe=pypy # pypy3 or pypy
 maj=7
 min=2
-rev=0
+rev=0rc0
 
 
 branchname=release-pypy$pmaj.$pmin-v$maj.x # ==OR== release-v$maj.x  # ==OR== release-v$maj.$min.x
-tagname=release-candidate-pypy$pmaj.$pmin-v$maj.$min.$rev  # ==OR== release-$maj.$min
-# tagname=release-pypy$pmaj.$pmin-v$maj.$min.$rev  # ==OR== release-$maj.$min
+tagname=release-pypy$pmaj.$pmin-v$maj.$min.$rev  # ==OR== release-$maj.$min
 
 echo checking hg log -r $branchname
 hg log -r $branchname || exit 1

From pypy.commits at gmail.com  Tue Oct  1 17:03:51 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 01 Oct 2019 14:03:51 -0700 (PDT)
Subject: [pypy-commit] pypy default: corner case in the json decoder: like
 regular object maps, don't make the json
Message-ID: <5d93bf37.1c69fb81.4e3a3.3f0d@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r97705:38ede7e5cb5a
Date: 2019-10-01 22:59 +0200
http://bitbucket.org/pypy/pypy/changeset/38ede7e5cb5a/

Log:	corner case in the json decoder: like regular object maps, don't
	make the json maps arbitrarily huge

diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -69,6 +69,9 @@
     # hit in the cache
     STRING_CACHE_USEFULNESS_FACTOR = 4
 
+    # don't make arbitrarily huge maps
+    MAX_MAP_SIZE = 100
+
 
     def __init__(self, space, s):
         self.space = space
@@ -368,7 +371,7 @@
                 return w_res
             elif ch == ',':
                 i = self.skip_whitespace(i)
-                if currmap.is_state_blocked():
+                if currmap.is_state_blocked() or nextindex > self.MAX_MAP_SIZE:
                     self.scratch.append(values_w)  # can reuse next time
                     dict_w = self._switch_to_dict(currmap, values_w, nextindex)
                     return self.decode_object_dict(i, start, dict_w)
diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py
--- a/pypy/module/_pypyjson/test/test__pypyjson.py
+++ b/pypy/module/_pypyjson/test/test__pypyjson.py
@@ -469,6 +469,14 @@
         res = _pypyjson.loads(json)
         assert res == [{u'a': 1}, {u'a': 2}]
 
+    def test_huge_map(self):
+        import _pypyjson
+        import __pypy__
+        s = '{' + ",".join('"%s": %s' % (i, i) for i in range(200)) + '}'
+        res = _pypyjson.loads(s)
+        assert len(res) == 200
+        assert __pypy__.strategy(res) == "UnicodeDictStrategy"
+
     def test_tab_in_string_should_fail(self):
         import _pypyjson
         # http://json.org/JSON_checker/test/fail25.json

From pypy.commits at gmail.com  Wed Oct  2 03:47:51 2019
From: pypy.commits at gmail.com (mattip)
Date: Wed, 02 Oct 2019 00:47:51 -0700 (PDT)
Subject: [pypy-commit] pypy default: Added tag release-pypy3.6-v7.2.0rc1 for
 changeset 4d6761df14ff
Message-ID: <5d945627.1c69fb81.b44fd.5048@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r97706:51d5d7494591
Date: 2019-10-02 10:46 +0300
http://bitbucket.org/pypy/pypy/changeset/51d5d7494591/

Log:	Added tag release-pypy3.6-v7.2.0rc1 for changeset 4d6761df14ff

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -52,3 +52,4 @@
 8cdda8b8cdb8ff29d9e620cccd6c5edd2f2a23ec release-pypy2.7-v7.1.1
 85dae4fd5c234b482feff834c73e089872194541 release-pypy2.7-v7.2.0rc0
 7ffb92269488f37c707ce66076f50ffd8613f8e2 release-pypy3.6-v7.2.0rc0
+4d6761df14ffd6f38450f183ac1fad32c946c21b release-pypy3.6-v7.2.0rc1

From pypy.commits at gmail.com  Wed Oct  2 09:53:48 2019
From: pypy.commits at gmail.com (arigo)
Date: Wed, 02 Oct 2019 06:53:48 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue #3084
Message-ID: <5d94abec.1c69fb81.4a5d6.99a4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r97707:cd96ab5b8d1e
Date: 2019-10-02 15:51 +0200
http://bitbucket.org/pypy/pypy/changeset/cd96ab5b8d1e/

Log:	Issue #3084

	Fix compilation error when building revdb

diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
--- a/rpython/translator/revdb/src-revdb/revdb_include.h
+++ b/rpython/translator/revdb/src-revdb/revdb_include.h
@@ -285,6 +285,8 @@
 #define OP_GC_RAWREFCOUNT_NEXT_DEAD(r)   \
     r = rpy_reverse_db_rawrefcount_next_dead()
 
+#define OP_GC_INCREASE_ROOT_STACK_DEPTH(depth, r)   /* nothing */
+
 
 RPY_EXTERN void rpy_reverse_db_flush(void);  /* must be called with the lock */
 RPY_EXTERN void rpy_reverse_db_fetch(const char *file, int line);

From pypy.commits at gmail.com  Wed Oct  2 11:40:01 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 02 Oct 2019 08:40:01 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy2.7-v7.x: corner case in the json
 decoder: like regular object maps, don't make the json
Message-ID: <5d94c4d1.1c69fb81.20e2c.8264@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: release-pypy2.7-v7.x
Changeset: r97708:6fe1105c1e17
Date: 2019-10-01 22:59 +0200
http://bitbucket.org/pypy/pypy/changeset/6fe1105c1e17/

Log:	corner case in the json decoder: like regular object maps, don't
	make the json maps arbitrarily huge (grafted from
	38ede7e5cb5ad44fdc637b8d5ee5f15849f84dbb)

diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -69,6 +69,9 @@
     # hit in the cache
     STRING_CACHE_USEFULNESS_FACTOR = 4
 
+    # don't make arbitrarily huge maps
+    MAX_MAP_SIZE = 100
+
 
     def __init__(self, space, s):
         self.space = space
@@ -368,7 +371,7 @@
                 return w_res
             elif ch == ',':
                 i = self.skip_whitespace(i)
-                if currmap.is_state_blocked():
+                if currmap.is_state_blocked() or nextindex > self.MAX_MAP_SIZE:
                     self.scratch.append(values_w)  # can reuse next time
                     dict_w = self._switch_to_dict(currmap, values_w, nextindex)
                     return self.decode_object_dict(i, start, dict_w)
diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py
--- a/pypy/module/_pypyjson/test/test__pypyjson.py
+++ b/pypy/module/_pypyjson/test/test__pypyjson.py
@@ -469,6 +469,14 @@
         res = _pypyjson.loads(json)
         assert res == [{u'a': 1}, {u'a': 2}]
 
+    def test_huge_map(self):
+        import _pypyjson
+        import __pypy__
+        s = '{' + ",".join('"%s": %s' % (i, i) for i in range(200)) + '}'
+        res = _pypyjson.loads(s)
+        assert len(res) == 200
+        assert __pypy__.strategy(res) == "UnicodeDictStrategy"
+
     def test_tab_in_string_should_fail(self):
         import _pypyjson
         # http://json.org/JSON_checker/test/fail25.json

From pypy.commits at gmail.com  Wed Oct  2 11:40:03 2019
From: pypy.commits at gmail.com (arigo)
Date: Wed, 02 Oct 2019 08:40:03 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy2.7-v7.x: Issue #3084
Message-ID: <5d94c4d3.1c69fb81.716f5.9537@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-pypy2.7-v7.x
Changeset: r97709:6a59a88053cc
Date: 2019-10-02 15:51 +0200
http://bitbucket.org/pypy/pypy/changeset/6a59a88053cc/

Log:	Issue #3084

	Fix compilation error when building revdb (grafted from
	cd96ab5b8d1e4364105cb4a3c21a31b5dc1a5c87)

diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
--- a/rpython/translator/revdb/src-revdb/revdb_include.h
+++ b/rpython/translator/revdb/src-revdb/revdb_include.h
@@ -285,6 +285,8 @@
 #define OP_GC_RAWREFCOUNT_NEXT_DEAD(r)   \
     r = rpy_reverse_db_rawrefcount_next_dead()
 
+#define OP_GC_INCREASE_ROOT_STACK_DEPTH(depth, r)   /* nothing */
+
 
 RPY_EXTERN void rpy_reverse_db_flush(void);  /* must be called with the lock */
 RPY_EXTERN void rpy_reverse_db_fetch(const char *file, int line);

From pypy.commits at gmail.com  Wed Oct  2 11:40:05 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 02 Oct 2019 08:40:05 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy3.6-v7.x: corner case in the json
 decoder: like regular object maps, don't make the json
Message-ID: <5d94c4d5.1c69fb81.2bcce.2241@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: release-pypy3.6-v7.x
Changeset: r97710:bd5961409c41
Date: 2019-10-01 22:59 +0200
http://bitbucket.org/pypy/pypy/changeset/bd5961409c41/

Log:	corner case in the json decoder: like regular object maps, don't
	make the json maps arbitrarily huge (grafted from
	38ede7e5cb5ad44fdc637b8d5ee5f15849f84dbb)

diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -73,6 +73,9 @@
     # hit in the cache
     STRING_CACHE_USEFULNESS_FACTOR = 4
 
+    # don't make arbitrarily huge maps
+    MAX_MAP_SIZE = 100
+
 
     def __init__(self, space, s):
         self.space = space
@@ -369,7 +372,7 @@
                 return w_res
             elif ch == ',':
                 i = self.skip_whitespace(i)
-                if currmap.is_state_blocked():
+                if currmap.is_state_blocked() or nextindex > self.MAX_MAP_SIZE:
                     self.scratch.append(values_w)  # can reuse next time
                     dict_w = self._switch_to_dict(currmap, values_w, nextindex)
                     return self.decode_object_dict(i, start, dict_w)
diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py
--- a/pypy/module/_pypyjson/test/test__pypyjson.py
+++ b/pypy/module/_pypyjson/test/test__pypyjson.py
@@ -467,6 +467,14 @@
         res = _pypyjson.loads(json)
         assert res == [{u'a': 1}, {u'a': 2}]
 
+    def test_huge_map(self):
+        import _pypyjson
+        import __pypy__
+        s = '{' + ",".join('"%s": %s' % (i, i) for i in range(200)) + '}'
+        res = _pypyjson.loads(s)
+        assert len(res) == 200
+        assert __pypy__.strategy(res) == "UnicodeDictStrategy"
+
     def test_tab_in_string_should_fail(self):
         import _pypyjson
         # http://json.org/JSON_checker/test/fail25.json

From pypy.commits at gmail.com  Wed Oct  2 11:40:07 2019
From: pypy.commits at gmail.com (arigo)
Date: Wed, 02 Oct 2019 08:40:07 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy3.6-v7.x: Issue #3084
Message-ID: <5d94c4d7.1c69fb81.c62ef.33ed@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-pypy3.6-v7.x
Changeset: r97711:7fbabb23dfec
Date: 2019-10-02 15:51 +0200
http://bitbucket.org/pypy/pypy/changeset/7fbabb23dfec/

Log:	Issue #3084

	Fix compilation error when building revdb (grafted from
	cd96ab5b8d1e4364105cb4a3c21a31b5dc1a5c87)

diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
--- a/rpython/translator/revdb/src-revdb/revdb_include.h
+++ b/rpython/translator/revdb/src-revdb/revdb_include.h
@@ -285,6 +285,8 @@
 #define OP_GC_RAWREFCOUNT_NEXT_DEAD(r)   \
     r = rpy_reverse_db_rawrefcount_next_dead()
 
+#define OP_GC_INCREASE_ROOT_STACK_DEPTH(depth, r)   /* nothing */
+
 
 RPY_EXTERN void rpy_reverse_db_flush(void);  /* must be called with the lock */
 RPY_EXTERN void rpy_reverse_db_fetch(const char *file, int line);

From pypy.commits at gmail.com  Wed Oct  2 15:52:29 2019
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 02 Oct 2019 12:52:29 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6-asyncgen: Add space.callable_w() method
Message-ID: <5d94fffd.1c69fb81.696f3.1ec2@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6-asyncgen
Changeset: r97712:4fb21ab8a5c6
Date: 2019-10-02 20:50 +0100
http://bitbucket.org/pypy/pypy/changeset/4fb21ab8a5c6/

Log:	Add space.callable_w() method

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1232,8 +1232,11 @@
         from pypy.interpreter.generator import GeneratorIterator
         return isinstance(w_obj, GeneratorIterator)
 
+    def callable_w(self, w_obj):
+        return self.lookup(w_obj, "__call__") is not None
+
     def callable(self, w_obj):
-        return self.newbool(self.lookup(w_obj, "__call__") is not None)
+        return self.newbool(self.callable_w(w_obj))
 
     def issequence_w(self, w_obj):
         flag = self.type(w_obj).flag_map_or_seq

From pypy.commits at gmail.com  Wed Oct  2 15:52:31 2019
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 02 Oct 2019 12:52:31 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6-asyncgen: Make asyncgen_hooks thread-local
Message-ID: <5d94ffff.1c69fb81.11cbb.503a@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6-asyncgen
Changeset: r97713:647cc06b34f4
Date: 2019-10-02 20:51 +0100
http://bitbucket.org/pypy/pypy/changeset/647cc06b34f4/

Log:	Make asyncgen_hooks thread-local

diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -23,8 +23,9 @@
     # XXX [fijal] but they're not. is_being_profiled is guarded a bit all
     #     over the place as well as w_tracefunc
 
-    _immutable_fields_ = ['profilefunc?', 'w_tracefunc?',
-                          'w_coroutine_wrapper_fn?']
+    _immutable_fields_ = [
+        'profilefunc?', 'w_tracefunc?', 'w_coroutine_wrapper_fn?',
+        'w_asyncgen_firstiter_fn?', 'w_asyncgen_finalizer_fn?']
 
     def __init__(self, space):
         self.space = space
@@ -41,6 +42,8 @@
         self.thread_disappeared = False   # might be set to True after os.fork()
         self.w_coroutine_wrapper_fn = None
         self.in_coroutine_wrapper = False
+        self.w_asyncgen_firstiter_fn = None
+        self.w_asyncgen_finalizer_fn = None
 
     @staticmethod
     def _mark_thread_disappeared(space):
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -594,15 +594,12 @@
         if self.hooks_inited:
             return
         self.hooks_inited = True
-
-        self.w_finalizer = self.space.appexec([], '''():
-            import sys
-            hooks = sys.get_asyncgen_hooks()
-            return hooks.finalizer''')
+        ec = self.space.getexecutioncontext()
+        self.w_finalizer = ec.w_asyncgen_firstiter_fn
 
     def _finalize_(self):
         if self.frame is not None and self.frame.lastblock is not None:
-            if self.w_finalizer is not self.space.w_None:
+            if self.w_finalizer is not None:
                 # XXX: this is a hack to resurrect the weakref that was cleared
                 # before running _finalize_()
                 if self.space.config.translation.rweakref:
diff --git a/pypy/module/sys/app.py b/pypy/module/sys/app.py
--- a/pypy/module/sys/app.py
+++ b/pypy/module/sys/app.py
@@ -75,7 +75,7 @@
 If it is another kind of object, it will be printed and the system
 exit status will be one (i.e., failure)."""
     # note that we cannot simply use SystemExit(exitcode) here.
-    # in the default branch, we use "raise SystemExit, exitcode", 
+    # in the default branch, we use "raise SystemExit, exitcode",
     # which leads to an extra de-tupelizing
     # in normalize_exception, which is exactly like CPython's.
     if isinstance(exitcode, tuple):
@@ -109,7 +109,6 @@
 
 # This is tested in test_app_main.py
 class sysflags(metaclass=structseqtype):
-
     name = "sys.flags"
 
     debug = structseqfield(0)
@@ -130,29 +129,6 @@
 null__xoptions = {}
 
 
-class asyncgen_hooks(metaclass=structseqtype):
-    name = "asyncgen_hooks"
-
-    firstiter = structseqfield(0)
-    finalizer = structseqfield(1)
-
-# FIXME: make this thread-local
-_current_asyncgen_hooks = asyncgen_hooks((None, None))
-
-def get_asyncgen_hooks():
-    return _current_asyncgen_hooks
-
-_default_arg = object()
-
-def set_asyncgen_hooks(firstiter=_default_arg, finalizer=_default_arg):
-    global _current_asyncgen_hooks
-    if firstiter is _default_arg:
-        firstiter = _current_asyncgen_hooks.firstiter
-    if finalizer is _default_arg:
-        finalizer = _current_asyncgen_hooks.finalizer
-    _current_asyncgen_hooks = asyncgen_hooks((firstiter, finalizer))
-
-
 implementation = SimpleNamespace(
     name='pypy',
     version=sys.version_info,
diff --git a/pypy/module/sys/moduledef.py b/pypy/module/sys/moduledef.py
--- a/pypy/module/sys/moduledef.py
+++ b/pypy/module/sys/moduledef.py
@@ -95,6 +95,8 @@
 
         'get_coroutine_wrapper' : 'vm.get_coroutine_wrapper',
         'set_coroutine_wrapper' : 'vm.set_coroutine_wrapper',
+        'get_asyncgen_hooks'    : 'vm.get_asyncgen_hooks',
+        'set_asyncgen_hooks'    : 'vm.set_asyncgen_hooks',
 
         'is_finalizing'         : 'vm.is_finalizing',
         }
@@ -115,8 +117,6 @@
         'flags'                 : 'app.null_sysflags',
         '_xoptions'             : 'app.null__xoptions',
         'implementation'        : 'app.implementation',
-        'get_asyncgen_hooks'    : 'app.get_asyncgen_hooks',
-        'set_asyncgen_hooks'    : 'app.set_asyncgen_hooks',
 
         # these six attributes are here only during tests;
         # they are removed before translation
@@ -184,7 +184,7 @@
                     if w_file is w_stdout:
                         e.write_unraisable(space, '', w_file)
                     ret = -1
-        return ret 
+        return ret
 
     def _file_is_closed(self, space, w_file):
         try:
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -7,7 +7,7 @@
 
 from pypy.interpreter import gateway
 from pypy.interpreter.error import oefmt
-from pypy.interpreter.gateway import unwrap_spec
+from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
 
 
 # ____________________________________________________________
@@ -240,6 +240,14 @@
     suite_mask = structseqfield(12, "Bit mask identifying available product suites")
     product_type = structseqfield(13, "System product type")
     platform_version = structseqfield(14, "Diagnostic version number")
+
+
+class asyncgen_hooks(metaclass=structseqtype):
+    name = "asyncgen_hooks"
+
+    firstiter = structseqfield(0)
+    finalizer = structseqfield(1)
+
 ''')
 
 
@@ -286,7 +294,7 @@
 
 getsizeof_missing = """getsizeof(...)
     getsizeof(object, default) -> int
-    
+
     Return the size of object in bytes.
 
 sys.getsizeof(object, default) will always return default on PyPy, and
@@ -351,5 +359,45 @@
     else:
         raise oefmt(space.w_TypeError, "callable expected, got %T", w_wrapper)
 
+def get_asyncgen_hooks(space):
+    """get_asyncgen_hooks()
+
+Return a namedtuple of installed asynchronous generators hooks (firstiter, finalizer)."""
+    ec = space.getexecutioncontext()
+    w_firstiter = ec.w_asyncgen_firstiter_fn
+    if w_firstiter is None:
+        w_firstiter = space.w_None
+    w_finalizer = ec.w_asyncgen_finalizer_fn
+    if w_finalizer is None:
+        w_finalizer = space.w_None
+    w_asyncgen_hooks = app.wget(space, "asyncgen_hooks")
+    return space.call_function(
+        w_asyncgen_hooks,
+        space.newtuple([w_firstiter, w_finalizer]))
+
+# Note: the docstring is wrong on CPython
+def set_asyncgen_hooks(space, w_firstiter=None, w_finalizer=None):
+    """set_asyncgen_hooks(firstiter=None, finalizer=None)
+
+Set a finalizer for async generators objects."""
+    ec = space.getexecutioncontext()
+    if space.is_w(w_finalizer, space.w_None):
+        ec.w_asyncgen_finalizer_fn = None
+    elif w_finalizer is not None:
+        if space.callable_w(w_finalizer):
+            ec.w_asyncgen_finalizer_fn = w_finalizer
+        else:
+            raise oefmt(space.w_TypeError,
+                "callable finalizer expected, got %T", w_finalizer)
+    if space.is_w(w_firstiter, space.w_None):
+        ec.w_asyncgen_firstiter_fn = None
+    elif w_firstiter is not None:
+        if space.callable_w(w_firstiter):
+            ec.w_asyncgen_firstiter_fn = w_firstiter
+        else:
+            raise oefmt(space.w_TypeError,
+                "callable firstiter expected, got %T", w_firstiter)
+
+
 def is_finalizing(space):
     return space.newbool(space.sys.finalizing)

From pypy.commits at gmail.com  Thu Oct  3 07:44:59 2019
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 03 Oct 2019 04:44:59 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6-asyncgen: fix comment
Message-ID: <5d95df3b.1c69fb81.6b8e2.f6c8@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6-asyncgen
Changeset: r97714:bb53568cb83f
Date: 2019-10-03 12:43 +0100
http://bitbucket.org/pypy/pypy/changeset/bb53568cb83f/

Log:	fix comment

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1121,7 +1121,7 @@
         # sub-iterable first before continuing on the next bytecode.
         in_generator = self.get_generator()
         if in_generator is None:
-            # Issue #2768: rare case involving __del__ methods.
+            # Issue #2786: rare case involving __del__ methods.
             # XXX This is a workaround, not a proper solution.
             raise oefmt(self.space.w_RuntimeError,
                         "PyPy limitation: cannot use 'yield from' or 'await' "

From pypy.commits at gmail.com  Thu Oct  3 13:15:38 2019
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 03 Oct 2019 10:15:38 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6-asyncgen: test cleanup
Message-ID: <5d962cba.1c69fb81.fceb0.08aa@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6-asyncgen
Changeset: r97715:7c0d5129bf7b
Date: 2019-10-03 15:39 +0100
http://bitbucket.org/pypy/pypy/changeset/7c0d5129bf7b/

Log:	test cleanup

diff --git a/pypy/interpreter/test/apptest_coroutine.py b/pypy/interpreter/test/apptest_coroutine.py
--- a/pypy/interpreter/test/apptest_coroutine.py
+++ b/pypy/interpreter/test/apptest_coroutine.py
@@ -633,16 +633,12 @@
         a2 = g.aclose()
     sys.set_asyncgen_hooks(finalizer=_finalize)
     assert state == 0
-    try:
+    with pytest.raises(StopIteration):
         a.send(None)
-    except StopIteration:
-        pass
     assert a2.send(None) == 'coro'
     assert state == 1
-    try:
+    with pytest.raises(StopIteration):
         a2.send(None)
-    except StopIteration:
-        pass
     assert state == 2
     sys.set_asyncgen_hooks(None, None)
 

From pypy.commits at gmail.com  Thu Oct  3 13:15:40 2019
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 03 Oct 2019 10:15:40 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6-asyncgen: Begin refactoring handling of
 .w_yielded_from
Message-ID: <5d962cbc.1c69fb81.33933.c7f4@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6-asyncgen
Changeset: r97716:d157cd5db6ca
Date: 2019-10-03 16:35 +0100
http://bitbucket.org/pypy/pypy/changeset/d157cd5db6ca/

Log:	Begin refactoring handling of .w_yielded_from

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -11,7 +11,7 @@
 class GeneratorOrCoroutine(W_Root):
     _immutable_fields_ = ['pycode']
 
-    w_yielded_from = None
+    _w_yielded_from = None
 
     def __init__(self, frame, name=None, qualname=None):
         self.space = frame.space
@@ -153,9 +153,9 @@
         # Called from execute_frame() just before resuming the bytecode
         # interpretation.
         space = self.space
-        w_yf = self.w_yielded_from
+        w_yf = self.get_delegate()
         if w_yf is not None:
-            self.w_yielded_from = None
+            self.set_delegate(None)
             try:
                 self.next_yield_from(frame, w_yf, w_arg_or_err)
             except OperationError as operr:
@@ -179,6 +179,12 @@
         else:
             return r_uint(0)
 
+    def get_delegate(self):
+        return self._w_yielded_from
+
+    def set_delegate(self, w_delegate):
+        self._w_yielded_from = w_delegate
+
     def next_yield_from(self, frame, w_yf, w_inputvalue_or_err):
         """Fetch the next item of the current 'yield from', push it on
         the frame stack, and raises Yield.  If there isn't one, push
@@ -209,7 +215,7 @@
             return
         else:
             frame.pushvalue(w_retval)
-            self.w_yielded_from = w_yf
+            self.set_delegate(w_yf)
             raise Yield
 
     def _leak_stopiteration(self, e):
@@ -259,8 +265,8 @@
         operr = OperationError(w_type, w_val, tb)
         operr.normalize_exception(space)
 
-        # note: w_yielded_from is always None if 'self.running'
-        if (self.w_yielded_from is not None and
+        # note: _w_yielded_from is always None if 'self.running'
+        if (self.get_delegate() is not None and
                     operr.match(space, space.w_GeneratorExit)):
             try:
                 self._gen_close_iter(space)
@@ -276,8 +282,8 @@
 
     def _gen_close_iter(self, space):
         assert not self.running
-        w_yf = self.w_yielded_from
-        self.w_yielded_from = None
+        w_yf = self.get_delegate()
+        self.set_delegate(None)
         self.running = True
         try:
             gen_close_iter(space, w_yf)
@@ -290,8 +296,8 @@
             return     # nothing to do in this case
         space = self.space
         operr = None
-        # note: w_yielded_from is always None if 'self.running'
-        w_yf = self.w_yielded_from
+        # note: _w_yielded_from is always None if 'self.running'
+        w_yf = self.get_delegate()
         if w_yf is not None:
             try:
                 self._gen_close_iter(space)
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1556,7 +1556,7 @@
         w_iterable = self.popvalue()
         w_iter = get_awaitable_iter(self.space, w_iterable)
         if isinstance(w_iter, Coroutine):
-            if w_iter.w_yielded_from is not None:
+            if w_iter.get_delegate() is not None:
                 # 'w_iter' is a coroutine object that is being awaited,
                 # '.w_yielded_from' is the current awaitable being awaited on.
                 raise oefmt(self.space.w_RuntimeError,
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -838,7 +838,7 @@
     gi_running = interp_attrproperty('running', cls=GeneratorIterator, wrapfn="newbool"),
     gi_frame   = GetSetProperty(GeneratorIterator.descr_gicr_frame),
     gi_code    = interp_attrproperty_w('pycode', cls=GeneratorIterator),
-    gi_yieldfrom=interp_attrproperty_w('w_yielded_from', cls=GeneratorIterator),
+    gi_yieldfrom=GetSetProperty(GeneratorIterator.get_delegate),
     __name__   = GetSetProperty(GeneratorIterator.descr__name__,
                                 GeneratorIterator.descr_set__name__),
     __qualname__ = GetSetProperty(GeneratorIterator.descr__qualname__,
@@ -862,7 +862,7 @@
     cr_running = interp_attrproperty('running', cls=Coroutine, wrapfn="newbool"),
     cr_frame   = GetSetProperty(Coroutine.descr_gicr_frame),
     cr_code    = interp_attrproperty_w('pycode', cls=Coroutine),
-    cr_await   = interp_attrproperty_w('w_yielded_from', cls=Coroutine),
+    cr_await=GetSetProperty(Coroutine.get_delegate),
     __name__   = GetSetProperty(Coroutine.descr__name__,
                                 Coroutine.descr_set__name__,
                                 doc="name of the coroutine"),
@@ -890,7 +890,7 @@
     ag_running = interp_attrproperty('running', cls=AsyncGenerator, wrapfn="newbool"),
     ag_frame   = GetSetProperty(AsyncGenerator.descr_gicr_frame),
     ag_code    = interp_attrproperty_w('pycode', cls=AsyncGenerator),
-    ag_await   = interp_attrproperty_w('w_yielded_from', cls=AsyncGenerator),
+    ag_await=GetSetProperty(AsyncGenerator.get_delegate),
     __name__   = GetSetProperty(AsyncGenerator.descr__name__,
                                 AsyncGenerator.descr_set__name__,
                                 doc="name of the async generator"),

From pypy.commits at gmail.com  Thu Oct  3 13:15:42 2019
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 03 Oct 2019 10:15:42 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6-asyncgen: Store w_yf on the frame instead
 of the generator
Message-ID: <5d962cbe.1c69fb81.11cbb.7115@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6-asyncgen
Changeset: r97717:b1d846ebee27
Date: 2019-10-03 16:52 +0100
http://bitbucket.org/pypy/pypy/changeset/b1d846ebee27/

Log:	Store w_yf on the frame instead of the generator

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -11,8 +11,6 @@
 class GeneratorOrCoroutine(W_Root):
     _immutable_fields_ = ['pycode']
 
-    _w_yielded_from = None
-
     def __init__(self, frame, name=None, qualname=None):
         self.space = frame.space
         self.frame = frame     # turned into None when frame_finished_execution
@@ -180,10 +178,10 @@
             return r_uint(0)
 
     def get_delegate(self):
-        return self._w_yielded_from
+        return self.frame.w_yielding_from
 
     def set_delegate(self, w_delegate):
-        self._w_yielded_from = w_delegate
+        self.frame.w_yielding_from = w_delegate
 
     def next_yield_from(self, frame, w_yf, w_inputvalue_or_err):
         """Fetch the next item of the current 'yield from', push it on
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -68,6 +68,7 @@
     frame_finished_execution = False
     f_generator_wref         = rweakref.dead_ref  # for generators/coroutines
     f_generator_nowref       = None               # (only one of the two attrs)
+    w_yielding_from = None
     last_instr               = -1
     f_backref                = jit.vref_None
 

From pypy.commits at gmail.com  Thu Oct  3 13:15:44 2019
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 03 Oct 2019 10:15:44 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6-asyncgen: Allow YIELD_FROM to run even if
 the generator owning the frame died
Message-ID: <5d962cc0.1c69fb81.9134d.00d6@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6-asyncgen
Changeset: r97718:c2f8aa002dda
Date: 2019-10-03 18:14 +0100
http://bitbucket.org/pypy/pypy/changeset/c2f8aa002dda/

Log:	Allow YIELD_FROM to run even if the generator owning the frame died

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -155,7 +155,7 @@
         if w_yf is not None:
             self.set_delegate(None)
             try:
-                self.next_yield_from(frame, w_yf, w_arg_or_err)
+                frame.next_yield_from(w_yf, w_arg_or_err)
             except OperationError as operr:
                 operr.record_context(space, space.getexecutioncontext())
                 return frame.handle_generator_error(operr)
@@ -178,43 +178,27 @@
             return r_uint(0)
 
     def get_delegate(self):
+        if self.frame is None:
+            return None
         return self.frame.w_yielding_from
 
+    def descr_delegate(self, space):
+        w_yf = self.get_delegate()
+        if w_yf is None:
+            return space.w_None
+        return w_yf
+
     def set_delegate(self, w_delegate):
         self.frame.w_yielding_from = w_delegate
 
+
+
     def next_yield_from(self, frame, w_yf, w_inputvalue_or_err):
         """Fetch the next item of the current 'yield from', push it on
         the frame stack, and raises Yield.  If there isn't one, push
         w_stopiteration_value and returns.  May also just raise.
         """
-        space = self.space
-        try:
-            if isinstance(w_yf, GeneratorOrCoroutine):
-                w_retval = w_yf.send_ex(w_inputvalue_or_err)
-            elif isinstance(w_yf, AsyncGenASend):   # performance only
-                w_retval = w_yf.do_send(w_inputvalue_or_err)
-            elif space.is_w(w_inputvalue_or_err, space.w_None):
-                w_retval = space.next(w_yf)
-            else:
-                w_retval = delegate_to_nongen(space, w_yf, w_inputvalue_or_err)
-        except OperationError as e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            frame._report_stopiteration_sometimes(w_yf, e)
-            try:
-                w_stop_value = space.getattr(e.get_w_value(space),
-                                             space.newtext("value"))
-            except OperationError as e:
-                if not e.match(space, space.w_AttributeError):
-                    raise
-                w_stop_value = space.w_None
-            frame.pushvalue(w_stop_value)
-            return
-        else:
-            frame.pushvalue(w_retval)
-            self.set_delegate(w_yf)
-            raise Yield
+        frame.next_yield_from(w_yf, w_inputvalue_or_err)
 
     def _leak_stopiteration(self, e):
         # Check for __future__ generator_stop and conditionally turn
@@ -493,27 +477,6 @@
         else:
             space.call_function(w_close)
 
-def delegate_to_nongen(space, w_yf, w_inputvalue_or_err):
-    # invoke a "send" or "throw" by method name to a non-generator w_yf
-    if isinstance(w_inputvalue_or_err, SApplicationException):
-        operr = w_inputvalue_or_err.operr
-        try:
-            w_meth = space.getattr(w_yf, space.newtext("throw"))
-        except OperationError as e:
-            if not e.match(space, space.w_AttributeError):
-                raise
-            raise operr
-        # bah, CPython calls here with the exact same arguments as
-        # originally passed to throw().  In our case it is far removed.
-        # Let's hope nobody will complain...
-        operr.normalize_exception(space)
-        w_exc = operr.w_type
-        w_val = operr.get_w_value(space)
-        w_tb  = operr.get_w_traceback(space)
-        return space.call_function(w_meth, w_exc, w_val, w_tb)
-    else:
-        return space.call_method(w_yf, "send", w_inputvalue_or_err)
-
 def gen_is_coroutine(w_obj):
     return (isinstance(w_obj, GeneratorIterator) and
             (w_obj.pycode.co_flags & consts.CO_ITERABLE_COROUTINE) != 0)
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1113,24 +1113,51 @@
             self.pushvalue(w_value)
         raise Yield
 
+    def next_yield_from(self, w_yf, w_inputvalue_or_err):
+        """Fetch the next item of the current 'yield from', push it on
+        the frame stack, and raises Yield.  If there isn't one, push
+        w_stopiteration_value and returns.  May also just raise.
+        """
+        from pypy.interpreter.generator import (
+            GeneratorOrCoroutine, AsyncGenASend)
+        space = self.space
+        try:
+            if isinstance(w_yf, GeneratorOrCoroutine):
+                w_retval = w_yf.send_ex(w_inputvalue_or_err)
+            elif isinstance(w_yf, AsyncGenASend):   # performance only
+                w_retval = w_yf.do_send(w_inputvalue_or_err)
+            elif space.is_w(w_inputvalue_or_err, space.w_None):
+                w_retval = space.next(w_yf)
+            else:
+                w_retval = delegate_to_nongen(space, w_yf, w_inputvalue_or_err)
+        except OperationError as e:
+            if not e.match(space, space.w_StopIteration):
+                raise
+            self._report_stopiteration_sometimes(w_yf, e)
+            try:
+                w_stop_value = space.getattr(e.get_w_value(space),
+                                             space.newtext("value"))
+            except OperationError as e:
+                if not e.match(space, space.w_AttributeError):
+                    raise
+                w_stop_value = space.w_None
+            self.pushvalue(w_stop_value)
+            return
+        else:
+            self.pushvalue(w_retval)
+            self.w_yielding_from = w_yf
+            raise Yield
+
     def YIELD_FROM(self, oparg, next_instr):
         # Unlike CPython, we handle this not by repeating the same
         # bytecode over and over until the inner iterator is exhausted.
         # Instead, we directly set the generator's w_yielded_from.
         # This asks generator.resume_execute_frame() to exhaust that
         # sub-iterable first before continuing on the next bytecode.
-        in_generator = self.get_generator()
-        if in_generator is None:
-            # Issue #2786: rare case involving __del__ methods.
-            # XXX This is a workaround, not a proper solution.
-            raise oefmt(self.space.w_RuntimeError,
-                        "PyPy limitation: cannot use 'yield from' or 'await' "
-                        "in a generator/coroutine that has been partially "
-                        "deallocated already, typically seen via __del__")
         w_inputvalue = self.popvalue()    # that's always w_None, actually
         w_gen = self.popvalue()
         #
-        in_generator.next_yield_from(self, w_gen, w_inputvalue)
+        self.next_yield_from(w_gen, w_inputvalue)
         # Common case: the call above raises Yield.
         # If instead the iterable is empty, next_yield_from() pushed the
         # final result and returns.  In that case, we can just continue
@@ -1694,6 +1721,27 @@
         else:
             self.MISSING_OPCODE(oparg, next_instr)
 
+def delegate_to_nongen(space, w_yf, w_inputvalue_or_err):
+    # invoke a "send" or "throw" by method name to a non-generator w_yf
+    if isinstance(w_inputvalue_or_err, SApplicationException):
+        operr = w_inputvalue_or_err.operr
+        try:
+            w_meth = space.getattr(w_yf, space.newtext("throw"))
+        except OperationError as e:
+            if not e.match(space, space.w_AttributeError):
+                raise
+            raise operr
+        # bah, CPython calls here with the exact same arguments as
+        # originally passed to throw().  In our case it is far removed.
+        # Let's hope nobody will complain...
+        operr.normalize_exception(space)
+        w_exc = operr.w_type
+        w_val = operr.get_w_value(space)
+        w_tb = operr.get_w_traceback(space)
+        return space.call_function(w_meth, w_exc, w_val, w_tb)
+    else:
+        return space.call_method(w_yf, "send", w_inputvalue_or_err)
+
 
 ### ____________________________________________________________ ###
 
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -838,7 +838,7 @@
     gi_running = interp_attrproperty('running', cls=GeneratorIterator, wrapfn="newbool"),
     gi_frame   = GetSetProperty(GeneratorIterator.descr_gicr_frame),
     gi_code    = interp_attrproperty_w('pycode', cls=GeneratorIterator),
-    gi_yieldfrom=GetSetProperty(GeneratorIterator.get_delegate),
+    gi_yieldfrom=GetSetProperty(GeneratorIterator.descr_delegate),
     __name__   = GetSetProperty(GeneratorIterator.descr__name__,
                                 GeneratorIterator.descr_set__name__),
     __qualname__ = GetSetProperty(GeneratorIterator.descr__qualname__,
@@ -862,7 +862,7 @@
     cr_running = interp_attrproperty('running', cls=Coroutine, wrapfn="newbool"),
     cr_frame   = GetSetProperty(Coroutine.descr_gicr_frame),
     cr_code    = interp_attrproperty_w('pycode', cls=Coroutine),
-    cr_await=GetSetProperty(Coroutine.get_delegate),
+    cr_await=GetSetProperty(Coroutine.descr_delegate),
     __name__   = GetSetProperty(Coroutine.descr__name__,
                                 Coroutine.descr_set__name__,
                                 doc="name of the coroutine"),
@@ -890,7 +890,7 @@
     ag_running = interp_attrproperty('running', cls=AsyncGenerator, wrapfn="newbool"),
     ag_frame   = GetSetProperty(AsyncGenerator.descr_gicr_frame),
     ag_code    = interp_attrproperty_w('pycode', cls=AsyncGenerator),
-    ag_await=GetSetProperty(AsyncGenerator.get_delegate),
+    ag_await=GetSetProperty(AsyncGenerator.descr_delegate),
     __name__   = GetSetProperty(AsyncGenerator.descr__name__,
                                 AsyncGenerator.descr_set__name__,
                                 doc="name of the async generator"),

From pypy.commits at gmail.com  Fri Oct  4 04:03:00 2019
From: pypy.commits at gmail.com (stefanor)
Date: Fri, 04 Oct 2019 01:03:00 -0700 (PDT)
Subject: [pypy-commit] pypy default: Handle ProcessorAutodetectError in
 _pypyjson.simd
Message-ID: <5d96fcb4.1c69fb81.9a21d.6ca2@mx.google.com>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: 
Changeset: r97719:f3edc9623b51
Date: 2019-10-04 10:58 +0300
http://bitbucket.org/pypy/pypy/changeset/f3edc9623b51/

Log:	Handle ProcessorAutodetectError in _pypyjson.simd

diff --git a/pypy/module/_pypyjson/simd.py b/pypy/module/_pypyjson/simd.py
--- a/pypy/module/_pypyjson/simd.py
+++ b/pypy/module/_pypyjson/simd.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib import objectmodel, unroll
 from rpython.rlib.rarithmetic import r_uint, intmask, LONG_BIT
-from rpython.jit.backend.detect_cpu import autodetect
+from rpython.jit.backend.detect_cpu import autodetect, ProcessorAutodetectError
 
 # accelerators for string operations using simd on regular word sizes (*not*
 # SSE instructions). this style is sometimes called SWAR (SIMD Within A
@@ -15,8 +15,11 @@
     WORD_SIZE = 8
     EVERY_BYTE_ONE = 0x0101010101010101
     EVERY_BYTE_HIGHEST_BIT = 0x8080808080808080
-    if autodetect() == "x86-64":
-        USE_SIMD = True
+    try:
+        if autodetect() == "x86-64":
+            USE_SIMD = True
+    except ProcessorAutodetectError:
+        pass
 else:
     WORD_SIZE = 4
     EVERY_BYTE_ONE = 0x01010101

From pypy.commits at gmail.com  Fri Oct  4 05:53:33 2019
From: pypy.commits at gmail.com (stefanor)
Date: Fri, 04 Oct 2019 02:53:33 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy3.6-v7.x: Handle
 ProcessorAutodetectError in _pypyjson.simd
Message-ID: <5d97169d.1c69fb81.1b5c1.ea74@mx.google.com>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: release-pypy3.6-v7.x
Changeset: r97720:6042bdab85aa
Date: 2019-10-04 10:58 +0300
http://bitbucket.org/pypy/pypy/changeset/6042bdab85aa/

Log:	Handle ProcessorAutodetectError in _pypyjson.simd (grafted from
	f3edc9623b51656f56f4a855327e77103050ef9e)

diff --git a/pypy/module/_pypyjson/simd.py b/pypy/module/_pypyjson/simd.py
--- a/pypy/module/_pypyjson/simd.py
+++ b/pypy/module/_pypyjson/simd.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib import objectmodel, unroll
 from rpython.rlib.rarithmetic import r_uint, intmask, LONG_BIT
-from rpython.jit.backend.detect_cpu import autodetect
+from rpython.jit.backend.detect_cpu import autodetect, ProcessorAutodetectError
 
 # accelerators for string operations using simd on regular word sizes (*not*
 # SSE instructions). this style is sometimes called SWAR (SIMD Within A
@@ -15,8 +15,11 @@
     WORD_SIZE = 8
     EVERY_BYTE_ONE = 0x0101010101010101
     EVERY_BYTE_HIGHEST_BIT = 0x8080808080808080
-    if autodetect() == "x86-64":
-        USE_SIMD = True
+    try:
+        if autodetect() == "x86-64":
+            USE_SIMD = True
+    except ProcessorAutodetectError:
+        pass
 else:
     WORD_SIZE = 4
     EVERY_BYTE_ONE = 0x01010101

From pypy.commits at gmail.com  Fri Oct  4 05:53:36 2019
From: pypy.commits at gmail.com (stefanor)
Date: Fri, 04 Oct 2019 02:53:36 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy2.7-v7.x: Handle
 ProcessorAutodetectError in _pypyjson.simd
Message-ID: <5d9716a0.1c69fb81.c26ea.371f@mx.google.com>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: release-pypy2.7-v7.x
Changeset: r97721:73aff81440f6
Date: 2019-10-04 10:58 +0300
http://bitbucket.org/pypy/pypy/changeset/73aff81440f6/

Log:	Handle ProcessorAutodetectError in _pypyjson.simd (grafted from
	f3edc9623b51656f56f4a855327e77103050ef9e)

diff --git a/pypy/module/_pypyjson/simd.py b/pypy/module/_pypyjson/simd.py
--- a/pypy/module/_pypyjson/simd.py
+++ b/pypy/module/_pypyjson/simd.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib import objectmodel, unroll
 from rpython.rlib.rarithmetic import r_uint, intmask, LONG_BIT
-from rpython.jit.backend.detect_cpu import autodetect
+from rpython.jit.backend.detect_cpu import autodetect, ProcessorAutodetectError
 
 # accelerators for string operations using simd on regular word sizes (*not*
 # SSE instructions). this style is sometimes called SWAR (SIMD Within A
@@ -15,8 +15,11 @@
     WORD_SIZE = 8
     EVERY_BYTE_ONE = 0x0101010101010101
     EVERY_BYTE_HIGHEST_BIT = 0x8080808080808080
-    if autodetect() == "x86-64":
-        USE_SIMD = True
+    try:
+        if autodetect() == "x86-64":
+            USE_SIMD = True
+    except ProcessorAutodetectError:
+        pass
 else:
     WORD_SIZE = 4
     EVERY_BYTE_ONE = 0x01010101

From pypy.commits at gmail.com  Fri Oct  4 13:03:36 2019
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 04 Oct 2019 10:03:36 -0700 (PDT)
Subject: [pypy-commit] pypy default: Add _PyDict_GetItemWithError (part of
 the public API in py3)
Message-ID: <5d977b68.1c69fb81.9b336.662e@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r97722:d663ce56919c
Date: 2019-10-04 18:02 +0100
http://bitbucket.org/pypy/pypy/changeset/d663ce56919c/

Log:	Add _PyDict_GetItemWithError (part of the public API in py3)

diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -80,6 +80,13 @@
     # XXX this is wrong with IntMutableCell.  Hope it works...
     return w_dict.getitem(w_key)
 
+ at cpython_api([PyObject, PyObject], PyObject, result_borrowed=True)
+def _PyDict_GetItemWithError(space, w_dict, w_key):
+    # Like PyDict_GetItem(), but doesn't swallow the error
+    if not isinstance(w_dict, W_DictMultiObject):
+        PyErr_BadInternalCall(space)
+    return w_dict.getitem(w_key)
+
 @cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)
 def PyDict_SetItem(space, w_dict, w_key, w_obj):
     if not isinstance(w_dict, W_DictMultiObject):
diff --git a/pypy/module/cpyext/test/test_dictobject.py b/pypy/module/cpyext/test/test_dictobject.py
--- a/pypy/module/cpyext/test/test_dictobject.py
+++ b/pypy/module/cpyext/test/test_dictobject.py
@@ -173,6 +173,26 @@
             ])
         assert module.dict_proxy({'a': 1, 'b': 2}) == 2
 
+    def test_getitemwitherror(self):
+        module = self.import_extension('foo', [
+            ("dict_getitem", "METH_VARARGS",
+             """
+             PyObject *d, *key, *result;
+             if (!PyArg_ParseTuple(args, "OO", &d, &key)) {
+                return NULL;
+             }
+             result = _PyDict_GetItemWithError(d, key);
+             if (result == NULL && !PyErr_Occurred())
+                Py_RETURN_NONE;
+             Py_XINCREF(result);
+             return result;
+             """)])
+        d = {'foo': 'bar'}
+        assert module.dict_getitem(d, 'foo') == 'bar'
+        assert module.dict_getitem(d, 'missing') is None
+        with raises(TypeError):
+            module.dict_getitem(d, [])
+
     def test_update(self):
         module = self.import_extension('foo', [
             ("update", "METH_VARARGS",

From pypy.commits at gmail.com  Fri Oct  4 13:30:57 2019
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 04 Oct 2019 10:30:57 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: hg merge default
Message-ID: <5d9781d1.1c69fb81.92d17.2137@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6
Changeset: r97723:a8752c53c0d2
Date: 2019-10-04 18:16 +0100
http://bitbucket.org/pypy/pypy/changeset/a8752c53c0d2/

Log:	hg merge default

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -50,4 +50,6 @@
 de061d87e39c7df4e436974096d7982c676a859d release-pypy3.6-v7.1.0
 784b254d669919c872a505b807db8462b6140973 release-pypy3.6-v7.1.1
 8cdda8b8cdb8ff29d9e620cccd6c5edd2f2a23ec release-pypy2.7-v7.1.1
-
+85dae4fd5c234b482feff834c73e089872194541 release-pypy2.7-v7.2.0rc0
+7ffb92269488f37c707ce66076f50ffd8613f8e2 release-pypy3.6-v7.2.0rc0
+4d6761df14ffd6f38450f183ac1fad32c946c21b release-pypy3.6-v7.2.0rc1
diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -73,6 +73,9 @@
     # hit in the cache
     STRING_CACHE_USEFULNESS_FACTOR = 4
 
+    # don't make arbitrarily huge maps
+    MAX_MAP_SIZE = 100
+
 
     def __init__(self, space, s):
         self.space = space
@@ -369,7 +372,7 @@
                 return w_res
             elif ch == ',':
                 i = self.skip_whitespace(i)
-                if currmap.is_state_blocked():
+                if currmap.is_state_blocked() or nextindex > self.MAX_MAP_SIZE:
                     self.scratch.append(values_w)  # can reuse next time
                     dict_w = self._switch_to_dict(currmap, values_w, nextindex)
                     return self.decode_object_dict(i, start, dict_w)
diff --git a/pypy/module/_pypyjson/simd.py b/pypy/module/_pypyjson/simd.py
--- a/pypy/module/_pypyjson/simd.py
+++ b/pypy/module/_pypyjson/simd.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib import objectmodel, unroll
 from rpython.rlib.rarithmetic import r_uint, intmask, LONG_BIT
-from rpython.jit.backend.detect_cpu import autodetect
+from rpython.jit.backend.detect_cpu import autodetect, ProcessorAutodetectError
 
 # accelerators for string operations using simd on regular word sizes (*not*
 # SSE instructions). this style is sometimes called SWAR (SIMD Within A
@@ -15,8 +15,11 @@
     WORD_SIZE = 8
     EVERY_BYTE_ONE = 0x0101010101010101
     EVERY_BYTE_HIGHEST_BIT = 0x8080808080808080
-    if autodetect() == "x86-64":
-        USE_SIMD = True
+    try:
+        if autodetect() == "x86-64":
+            USE_SIMD = True
+    except ProcessorAutodetectError:
+        pass
 else:
     WORD_SIZE = 4
     EVERY_BYTE_ONE = 0x01010101
diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py
--- a/pypy/module/_pypyjson/test/test__pypyjson.py
+++ b/pypy/module/_pypyjson/test/test__pypyjson.py
@@ -467,6 +467,14 @@
         res = _pypyjson.loads(json)
         assert res == [{u'a': 1}, {u'a': 2}]
 
+    def test_huge_map(self):
+        import _pypyjson
+        import __pypy__
+        s = '{' + ",".join('"%s": %s' % (i, i) for i in range(200)) + '}'
+        res = _pypyjson.loads(s)
+        assert len(res) == 200
+        assert __pypy__.strategy(res) == "UnicodeDictStrategy"
+
     def test_tab_in_string_should_fail(self):
         import _pypyjson
         # http://json.org/JSON_checker/test/fail25.json
diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -80,6 +80,13 @@
     # XXX this is wrong with IntMutableCell.  Hope it works...
     return w_dict.getitem(w_key)
 
+ at cpython_api([PyObject, PyObject], PyObject, result_borrowed=True)
+def _PyDict_GetItemWithError(space, w_dict, w_key):
+    # Like PyDict_GetItem(), but doesn't swallow the error
+    if not isinstance(w_dict, W_DictMultiObject):
+        PyErr_BadInternalCall(space)
+    return w_dict.getitem(w_key)
+
 @cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)
 def PyDict_SetItem(space, w_dict, w_key, w_obj):
     if not isinstance(w_dict, W_DictMultiObject):
diff --git a/pypy/module/cpyext/test/test_dictobject.py b/pypy/module/cpyext/test/test_dictobject.py
--- a/pypy/module/cpyext/test/test_dictobject.py
+++ b/pypy/module/cpyext/test/test_dictobject.py
@@ -175,6 +175,26 @@
             ])
         assert module.dict_proxy({'a': 1, 'b': 2}) == 2
 
+    def test_getitemwitherror(self):
+        module = self.import_extension('foo', [
+            ("dict_getitem", "METH_VARARGS",
+             """
+             PyObject *d, *key, *result;
+             if (!PyArg_ParseTuple(args, "OO", &d, &key)) {
+                return NULL;
+             }
+             result = _PyDict_GetItemWithError(d, key);
+             if (result == NULL && !PyErr_Occurred())
+                Py_RETURN_NONE;
+             Py_XINCREF(result);
+             return result;
+             """)])
+        d = {'foo': 'bar'}
+        assert module.dict_getitem(d, 'foo') == 'bar'
+        assert module.dict_getitem(d, 'missing') is None
+        with raises(TypeError):
+            module.dict_getitem(d, [])
+
     def test_setdefault(self):
         module = self.import_extension('foo', [
             ("setdefault", "METH_VARARGS",
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -36,6 +36,10 @@
 def prev_codepoint_pos_dont_look_inside(utf8, p):
     return rutf8.prev_codepoint_pos(utf8, p)
 
+ at jit.elidable
+def codepoint_at_pos_dont_look_inside(utf8, p):
+    return rutf8.codepoint_at_pos(utf8, p)
+
 
 class W_UnicodeObject(W_Root):
     import_from_mixin(StringMethods)
@@ -122,7 +126,7 @@
             raise oefmt(space.w_TypeError,
                          "ord() expected a character, but string of length %d "
                          "found", self._len())
-        return space.newint(rutf8.codepoint_at_pos(self._utf8, 0))
+        return space.newint(self.codepoint_at_pos_dont_look_inside(0))
 
     def _empty(self):
         return W_UnicodeObject.EMPTY
@@ -548,7 +552,7 @@
         if self._length == 0:
             return space.w_False
         if self._length == 1:
-            return space.newbool(func(rutf8.codepoint_at_pos(self._utf8, 0)))
+            return space.newbool(func(self.codepoint_at_pos_dont_look_inside(0)))
         else:
             return self._is_generic_loop(space, self._utf8, func_name)
 
@@ -1127,6 +1131,11 @@
             return pos - 1
         return prev_codepoint_pos_dont_look_inside(self._utf8, pos)
 
+    def codepoint_at_pos_dont_look_inside(self, pos):
+        if self.is_ascii():
+            return ord(self._utf8[pos])
+        return codepoint_at_pos_dont_look_inside(self._utf8, pos)
+
     @always_inline
     def _unwrap_and_search(self, space, w_sub, w_start, w_end, forward=True):
         w_sub = self.convert_arg_to_w_unicode(space, w_sub)
diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
--- a/rpython/translator/revdb/src-revdb/revdb_include.h
+++ b/rpython/translator/revdb/src-revdb/revdb_include.h
@@ -285,6 +285,8 @@
 #define OP_GC_RAWREFCOUNT_NEXT_DEAD(r)   \
     r = rpy_reverse_db_rawrefcount_next_dead()
 
+#define OP_GC_INCREASE_ROOT_STACK_DEPTH(depth, r)   /* nothing */
+
 
 RPY_EXTERN void rpy_reverse_db_flush(void);  /* must be called with the lock */
 RPY_EXTERN void rpy_reverse_db_fetch(const char *file, int line);

From pypy.commits at gmail.com  Fri Oct  4 13:30:59 2019
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 04 Oct 2019 10:30:59 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: Rename to PyDict_GetItemWithError (no
 leading underscore)
Message-ID: <5d9781d3.1c69fb81.2c32d.d9b5@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6
Changeset: r97724:6e344fc026c3
Date: 2019-10-04 18:29 +0100
http://bitbucket.org/pypy/pypy/changeset/6e344fc026c3/

Log:	Rename to PyDict_GetItemWithError (no leading underscore)

diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -81,8 +81,11 @@
     return w_dict.getitem(w_key)
 
 @cpython_api([PyObject, PyObject], PyObject, result_borrowed=True)
-def _PyDict_GetItemWithError(space, w_dict, w_key):
-    # Like PyDict_GetItem(), but doesn't swallow the error
+def PyDict_GetItemWithError(space, w_dict, w_key):
+    """Variant of PyDict_GetItem() that does not suppress
+    exceptions. Return NULL with an exception set if an exception
+    occurred.  Return NULL without an exception set if the key
+    wasn't present."""
     if not isinstance(w_dict, W_DictMultiObject):
         PyErr_BadInternalCall(space)
     return w_dict.getitem(w_key)
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -243,14 +243,6 @@
 def PyWrapper_New(space, w_d, w_self):
     raise NotImplementedError
 
- at cpython_api([PyObject, PyObject], PyObject)
-def PyDict_GetItemWithError(space, p, key):
-    """Variant of PyDict_GetItem() that does not suppress
-    exceptions. Return NULL with an exception set if an exception
-    occurred.  Return NULL without an exception set if the key
-    wasn't present."""
-    raise NotImplementedError
-
 @cpython_api([PyObject, PyObject, rffi.INT_real], rffi.INT_real, error=-1)
 def PyDict_MergeFromSeq2(space, a, seq2, override):
     """Update or merge into dictionary a, from the key-value pairs in seq2.
diff --git a/pypy/module/cpyext/test/test_dictobject.py b/pypy/module/cpyext/test/test_dictobject.py
--- a/pypy/module/cpyext/test/test_dictobject.py
+++ b/pypy/module/cpyext/test/test_dictobject.py
@@ -183,7 +183,7 @@
              if (!PyArg_ParseTuple(args, "OO", &d, &key)) {
                 return NULL;
              }
-             result = _PyDict_GetItemWithError(d, key);
+             result = PyDict_GetItemWithError(d, key);
              if (result == NULL && !PyErr_Occurred())
                 Py_RETURN_NONE;
              Py_XINCREF(result);

From pypy.commits at gmail.com  Fri Oct  4 16:20:50 2019
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 04 Oct 2019 13:20:50 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6-asyncgen: Move .resume_execute_frame() to
 PyFrame and simplify code
Message-ID: <5d97a9a2.1c69fb81.622ac.7563@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6-asyncgen
Changeset: r97727:ce7a9b715879
Date: 2019-10-04 21:19 +0100
http://bitbucket.org/pypy/pypy/changeset/ce7a9b715879/

Log:	Move .resume_execute_frame() to PyFrame and simplify code

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -82,7 +82,7 @@
                 operr = OperationError(space.w_StopIteration, space.w_None)
             raise operr
 
-        w_result = self._invoke_execute_frame(frame, w_arg_or_err)
+        w_result = self._invoke_execute_frame(w_arg_or_err)
         assert w_result is not None
 
         # if the frame is now marked as finished, it was RETURNed from
@@ -100,8 +100,9 @@
         else:
             return w_result     # YIELDed
 
-    def _invoke_execute_frame(self, frame, w_arg_or_err):
+    def _invoke_execute_frame(self, w_arg_or_err):
         space = self.space
+        frame = self.frame
         if self.running:
             raise oefmt(space.w_ValueError, "%s already executing", self.KIND)
         ec = space.getexecutioncontext()
@@ -147,36 +148,6 @@
             ec.set_sys_exc_info(current_exc_info)
         return w_result
 
-    def resume_execute_frame(self, frame, w_arg_or_err):
-        # Called from execute_frame() just before resuming the bytecode
-        # interpretation.
-        space = self.space
-        w_yf = self.get_delegate()
-        if w_yf is not None:
-            self.set_delegate(None)
-            try:
-                frame.next_yield_from(w_yf, w_arg_or_err)
-            except OperationError as operr:
-                operr.record_context(space, space.getexecutioncontext())
-                return frame.handle_generator_error(operr)
-            # Normal case: the call above raises Yield.
-            # We reach this point if the iterable is exhausted.
-            last_instr = jit.promote(frame.last_instr)
-            assert last_instr & 1 == 0
-            assert last_instr >= 0
-            return r_uint(last_instr + 2)
-
-        if isinstance(w_arg_or_err, SApplicationException):
-            return frame.handle_generator_error(w_arg_or_err.operr)
-
-        last_instr = jit.promote(frame.last_instr)
-        if last_instr != -1:
-            assert last_instr & 1 == 0
-            frame.pushvalue(w_arg_or_err)
-            return r_uint(last_instr + 2)
-        else:
-            return r_uint(0)
-
     def get_delegate(self):
         if self.frame is None:
             return None
@@ -191,15 +162,6 @@
     def set_delegate(self, w_delegate):
         self.frame.w_yielding_from = w_delegate
 
-
-
-    def next_yield_from(self, frame, w_yf, w_inputvalue_or_err):
-        """Fetch the next item of the current 'yield from', push it on
-        the frame stack, and raises Yield.  If there isn't one, push
-        w_stopiteration_value and returns.  May also just raise.
-        """
-        frame.next_yield_from(w_yf, w_inputvalue_or_err)
-
     def _leak_stopiteration(self, e):
         # Check for __future__ generator_stop and conditionally turn
         # a leaking StopIteration into RuntimeError (with its cause
@@ -363,7 +325,7 @@
     # generate 2 versions of the function and 2 jit drivers.
     def _create_unpack_into():
         jitdriver = jit.JitDriver(greens=['pycode'],
-                                  reds=['self', 'frame', 'results'],
+                                  reds='auto',
                                   name='unpack_into')
 
         def unpack_into(self, results):
@@ -374,12 +336,10 @@
                 return
             pycode = self.pycode
             while True:
-                jitdriver.jit_merge_point(self=self, frame=frame,
-                                          results=results, pycode=pycode)
+                jitdriver.jit_merge_point(pycode=pycode)
                 space = self.space
                 try:
-                    w_result = self._invoke_execute_frame(
-                                            frame, space.w_None)
+                    w_result = self._invoke_execute_frame(space.w_None)
                 except OperationError as e:
                     if not e.match(space, space.w_StopIteration):
                         raise
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -320,6 +320,37 @@
                 ec.in_coroutine_wrapper = False
         return w_gen
 
+    def resume_execute_frame(self, w_arg_or_err):
+        # Called from execute_frame() just before resuming the bytecode
+        # interpretation.
+        from pypy.interpreter.pyopcode import SApplicationException
+        space = self.space
+        w_yf = self.w_yielding_from
+        if w_yf is not None:
+            self.w_yielding_from = None
+            try:
+                self.next_yield_from(w_yf, w_arg_or_err)
+            except OperationError as operr:
+                operr.record_context(space, space.getexecutioncontext())
+                return self.handle_generator_error(operr)
+            # Normal case: the call above raises Yield.
+            # We reach this point if the iterable is exhausted.
+            last_instr = jit.promote(self.last_instr)
+            assert last_instr & 1 == 0
+            assert last_instr >= 0
+            return r_uint(last_instr + 2)
+
+        if isinstance(w_arg_or_err, SApplicationException):
+            return self.handle_generator_error(w_arg_or_err.operr)
+
+        last_instr = jit.promote(self.last_instr)
+        if last_instr != -1:
+            assert last_instr & 1 == 0
+            self.pushvalue(w_arg_or_err)
+            return r_uint(last_instr + 2)
+        else:
+            return r_uint(0)
+
     def execute_frame(self, in_generator=None, w_arg_or_err=None):
         """Execute this frame.  Main entry point to the interpreter.
         'in_generator' is non-None iff we are starting or resuming
@@ -348,8 +379,7 @@
                         assert self.last_instr == -1
                         next_instr = r_uint(0)
                     else:
-                        next_instr = in_generator.resume_execute_frame(
-                                                        self, w_arg_or_err)
+                        next_instr = self.resume_execute_frame(w_arg_or_err)
                 except pyopcode.Yield:
                     w_exitvalue = self.popvalue()
                 else:

From pypy.commits at gmail.com  Sat Oct  5 04:50:52 2019
From: pypy.commits at gmail.com (stevie_92)
Date: Sat, 05 Oct 2019 01:50:52 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-gc-cycle: Fixed issues with rrc tuples
Message-ID: <5d98596c.1c69fb81.3a337.9c12@mx.google.com>

Author: Stefan Beyer <home at sbeyer.at>
Branch: cpyext-gc-cycle
Changeset: r97728:6e21fe036218
Date: 2019-10-05 10:50 +0200
http://bitbucket.org/pypy/pypy/changeset/6e21fe036218/

Log:	Fixed issues with rrc tuples Implemented cpyext statistics

diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -8,6 +8,7 @@
 from rpython.rlib import rawrefcount
 from rpython.rlib.debug import debug_print
 import sys
+import time
 
 
 # Keep track of exceptions raised in cpyext for a particular execution
@@ -159,6 +160,10 @@
                 space.actionflag.register_periodic_action(action,
                     use_bytecode_counter=True)
             else:
+                module = space.builtin_modules['gc']
+                attribute = space.newtext('cpyext_durations')
+                space.setattr(module, attribute, space.newlist([]))
+
                 pyobj_dealloc_action = PyObjDeallocAction(space)
                 self.dealloc_trigger = lambda: pyobj_dealloc_action.fire()
 
@@ -310,12 +315,14 @@
         return True
 
 
-def _rawrefcount_perform(space):
+def _rawrefcount_perform(space): # TODO: measure time spent, make incremental??
     from pypy.interpreter.baseobjspace import W_Root
     from pypy.module.cpyext.pyobject import (PyObject, incref, decref,
                                              finalize, from_ref, cts)
     from pypy.module.cpyext.api import generic_cpy_call
 
+    start = time.time()
+
     while True:
         py_obj = rawrefcount.next_dead(PyObject)
         if not py_obj:
@@ -337,11 +344,11 @@
         pto = pyobj.c_ob_type
         if pto.c_tp_clear:
             incref(space, py_obj)
-            if pto and pto.c_tp_name:
-                tp_name = pto.c_tp_name
-                name = rffi.charp2str(cts.cast('char*', tp_name))
-                debug_print("tp_clear", pyobj, ": type", pto,
-                            ": name", name)
+            #if pto and pto.c_tp_name:
+            #    tp_name = pto.c_tp_name
+            #    name = rffi.charp2str(cts.cast('char*', tp_name))
+            #    debug_print("tp_clear", pyobj, ": type", pto,
+            #                ": name", name)
             generic_cpy_call(space, pto.c_tp_clear, pyobj)
             decref(space, py_obj)
         head = rawrefcount.cyclic_garbage_head(PyObject)
@@ -365,6 +372,11 @@
                   w_list)
     rawrefcount.end_garbage()
 
+    duration = time.time() - start
+    module = space.builtin_modules['gc']
+    durations = space.getattr(module, space.newtext('cpyext_durations'))
+    durations.append(space.newfloat(duration))
+
 class PyObjDeallocAction(executioncontext.AsyncAction):
     """An action that invokes _Py_Dealloc() on the dying PyObjects.
     """
diff --git a/rpython/memory/gc/rrc/incmark.py b/rpython/memory/gc/rrc/incmark.py
--- a/rpython/memory/gc/rrc/incmark.py
+++ b/rpython/memory/gc/rrc/incmark.py
@@ -258,7 +258,8 @@
                 addr = self.snapshot_refs[obj.refs_index + j]
                 obj_ref = llmemory.cast_adr_to_ptr(addr,
                                                    self.PYOBJ_SNAPSHOT_OBJ_PTR)
-                obj_ref.refcnt -= 1
+                if obj_ref != lltype.nullptr(self.PYOBJ_SNAPSHOT_OBJ):
+                    obj_ref.refcnt -= 1
 
         # now all rawrefcounted roots or live border objects have a
         # refcount > 0
@@ -299,7 +300,8 @@
                 addr = self.snapshot_refs[snapobj.refs_index + j]
                 obj_ref = llmemory.cast_adr_to_ptr(addr,
                                                    self.PYOBJ_SNAPSHOT_OBJ_PTR)
-                obj_ref.refcnt += 1
+                if obj_ref != lltype.nullptr(self.PYOBJ_SNAPSHOT_OBJ):
+                    obj_ref.refcnt += 1
             # mark recursively, if it is a pypyobj
             if snapobj.pypy_link <> 0:
                 intobj = snapobj.pypy_link
@@ -320,6 +322,10 @@
             total_refcnt += self._take_snapshot_count_gc(pygchdr)
             total_objs += 1
             pygchdr = pygchdr.c_gc_next
+        pygchdr = self.tuple_list.c_gc_next
+        while pygchdr <> self.tuple_list:
+            total_refcnt += self._take_snapshot_count_gc(pygchdr)
+            pygchdr = pygchdr.c_gc_next
         pygchdr = self.pyobj_isolate_old_list.c_gc_next
         while pygchdr <> self.pyobj_isolate_old_list:
             total_refcnt += self._take_snapshot_count_gc(pygchdr)
@@ -364,7 +370,10 @@
             pygchdr = self.pyobj_as_gc(pyobj)
             if (pygchdr <> lltype.nullptr(self.PYOBJ_GC_HDR) and
                 pygchdr.c_gc_refs != self.RAWREFCOUNT_REFS_UNTRACKED):
-                obj = self.snapshot_objs[pygchdr.c_gc_refs - 1]
+                if pygchdr.c_gc_refs > 0:
+                    obj = self.snapshot_objs[pygchdr.c_gc_refs - 1]
+                else:
+                    obj = lltype.nullptr(self.PYOBJ_SNAPSHOT_OBJ)
             else:
                 obj = self.snapshot_objs[pyobj.c_ob_pypy_link - 1]
             self.snapshot_refs[i] = llmemory.cast_ptr_to_adr(obj)
diff --git a/rpython/memory/gc/test/dot/free_cpython_tuple_1.dot b/rpython/memory/gc/test/dot/free_cpython_tuple_1.dot
--- a/rpython/memory/gc/test/dot/free_cpython_tuple_1.dot
+++ b/rpython/memory/gc/test/dot/free_cpython_tuple_1.dot
@@ -1,5 +1,5 @@
 digraph G {
-    "a" [type=C, alive=n, tuple=y];
+    "a" [type=C, alive=n, tuple=1];
     "b" [type=C, alive=n];
     "a" -> "b";
     "b" -> "a";
diff --git a/rpython/memory/gc/test/dot/free_cpython_tuple_1.dot b/rpython/memory/gc/test/dot/free_cpython_tuple_2.dot
copy from rpython/memory/gc/test/dot/free_cpython_tuple_1.dot
copy to rpython/memory/gc/test/dot/free_cpython_tuple_2.dot
--- a/rpython/memory/gc/test/dot/free_cpython_tuple_1.dot
+++ b/rpython/memory/gc/test/dot/free_cpython_tuple_2.dot
@@ -1,6 +1,5 @@
 digraph G {
-    "a" [type=C, alive=n, tuple=y];
-    "b" [type=C, alive=n];
+    "a" [type=C, alive=y, ext_refcnt=1];
+    "b" [type=C, alive=y, tuple=0];
     "a" -> "b";
-    "b" -> "a";
 }
diff --git a/rpython/memory/gc/test/dot/free_cpython_tuple_1.dot b/rpython/memory/gc/test/dot/free_cpython_tuple_3.dot
copy from rpython/memory/gc/test/dot/free_cpython_tuple_1.dot
copy to rpython/memory/gc/test/dot/free_cpython_tuple_3.dot
--- a/rpython/memory/gc/test/dot/free_cpython_tuple_1.dot
+++ b/rpython/memory/gc/test/dot/free_cpython_tuple_3.dot
@@ -1,6 +1,5 @@
 digraph G {
-    "a" [type=C, alive=n, tuple=y];
-    "b" [type=C, alive=n];
+    "a" [type=C, alive=y, ext_refcnt=1];
+    "b" [type=C, alive=y, tuple=2];
     "a" -> "b";
-    "b" -> "a";
 }
diff --git a/rpython/memory/gc/test/test_rawrefcount.py b/rpython/memory/gc/test/test_rawrefcount.py
--- a/rpython/memory/gc/test/test_rawrefcount.py
+++ b/rpython/memory/gc/test/test_rawrefcount.py
@@ -43,6 +43,7 @@
         self.pyobj_resurrect = {}
         self.pyobj_delete = {}
         self.is_pygc = []
+        self.tupletypes = []
 
         def rawrefcount_tp_traverse(obj, callback, args):
             refs = self.pyobj_refs[self.pyobjs.index(obj)]
@@ -73,13 +74,19 @@
                 return RAWREFCOUNT_FINALIZER_NONE
 
         def rawrefcount_tuple_maybe_untrack(obj):
-            #if foo:
-            #    gchdr = rawrefcount_pyobj_as_gc(obj)
-            #    next = gchdr.c_gc_next
-            #    next.c_gc_prev = gchdr.c_gc_prev
-            #    gchdr.c_gc_prev.c_gc_next = next
-            #    return 0
-            return 1 # TODO: add tests for 0 ("plain" tuple) and 2 (uninitialized)
+            index = self.pyobjs.index(obj)
+            if self.tupletypes[index] == '0':
+                gchdr = self.gcobjs[index]
+                next = gchdr.c_gc_next
+                next.c_gc_prev = gchdr.c_gc_prev
+                gchdr.c_gc_prev.c_gc_next = next
+                return 0
+            elif self.tupletypes[index] == '1':
+                return 1
+            elif self.tupletypes[index] == '2':
+                return 2
+            else:
+                assert False
 
         self.pyobj_list = lltype.malloc(PYOBJ_GC_HDR_PTR.TO, flavor='raw',
                                         immortal=True)
@@ -146,7 +153,7 @@
         return p1, p1ref, check_alive
 
     def _rawrefcount_pyobj(self, create_immortal=False, is_gc=True,
-                           tracked=True, tuple=tuple):
+                           tracked=True, tuple=False, tuple_type=None):
         r1 = lltype.malloc(PYOBJ_HDR, flavor='raw',
                            immortal=create_immortal)
         r1.c_ob_refcnt = 0
@@ -157,6 +164,7 @@
             self._rawrefcount_add_gc(tracked, tuple)
         else:
             self.gcobjs.append(lltype.nullptr(PYOBJ_GC_HDR))
+        self.tupletypes.append(tuple_type)
 
         self.pyobjs.append(r1)
         self.is_pygc.append(is_gc)
@@ -171,7 +179,7 @@
     def _rawrefcount_pair(self, intval, is_light=False, is_pyobj=False,
                           create_old=False, create_immortal=False,
                           rooted=False, force_external=False, is_gc=True,
-                          tracked=True, tuple=tuple):
+                          tracked=True, tuple=False, tuple_type=None):
         if is_light:
             rc = REFCNT_FROM_PYPY_LIGHT
         else:
@@ -207,6 +215,7 @@
             self._rawrefcount_add_gc(tracked, tuple)
         else:
             self.gcobjs.append(lltype.nullptr(PYOBJ_GC_HDR))
+        self.tupletypes.append(tuple_type)
 
         self.pyobjs.append(r1)
         self.is_pygc.append(is_gc)
@@ -558,7 +567,8 @@
             resurrect = attr['resurrect'] if 'resurrect' in attr else None
             delete = attr['delete'] if 'delete' in attr else None
             garbage = True if 'garbage' in attr else False
-            tuple = attr['tuple'] == "y" if 'tuple' in attr else False
+            tuple = True if 'tuple' in attr else False
+            tuple_type = attr['tuple'] if 'tuple' in attr else None
             gc = attr['gc'] == "y" if 'gc' in attr else True
             added = attr['added'] if 'added' in attr else None
             info = NodeInfo(type, alive, ext_refcnt, finalizer, resurrect,
@@ -569,7 +579,7 @@
                     add_pyobj_after_snap.append(nodes[name])
                 else:
                     r, raddr, check_alive = self._rawrefcount_pyobj(
-                        tracked=tracked, tuple=tuple)
+                        tracked=tracked, tuple=tuple, tuple_type=tuple_type)
                     r.c_ob_refcnt += ext_refcnt
                     nodes[name] = CPythonNode(r, raddr, check_alive, info)
             elif type == "P":
@@ -600,7 +610,7 @@
                         self._rawrefcount_pair(42 + i, rooted=rooted,
                                                create_old=True,
                                                tracked=tracked, tuple=tuple,
-                                               is_gc=gc)
+                                               tuple_type=tuple_type, is_gc=gc)
                     r.c_ob_refcnt += ext_refcnt
                     nodes[name] = BorderNode(p, pref, r, raddr, check_alive,
                                              info)

From pypy.commits at gmail.com  Sat Oct  5 07:49:37 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Sat, 05 Oct 2019 04:49:37 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: some more text
Message-ID: <5d988351.1c69fb81.be3e3.4465@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: extradoc
Changeset: r5963:73fb50383067
Date: 2019-10-05 13:49 +0200
http://bitbucket.org/pypy/extradoc/changeset/73fb50383067/

Log:	some more text

diff --git a/blog/draft/2019-10-json.rst b/blog/draft/2019-10-json.rst
--- a/blog/draft/2019-10-json.rst
+++ b/blog/draft/2019-10-json.rst
@@ -223,4 +223,119 @@
 (simplejson), with ujson, with the JSON parser of Node/V8 and with
 RapidJSON (in DOM mode).
 
-$$$
+
+I collected a number of medium-to-large JSON files to try the JSON
+parsers on:
+
+-  `Censys <https://censys.io/data>`__: A subset of the Censys port and
+   protocol scan data for websites in the Alexa top million domains
+-  `Gharchive <https://www.gharchive.org>`__: Github activity from
+   January 15-23, 2015 from Github Archive
+-  `Reddit <https://files.pushshift.io/reddit/comments/>`__: Reddit
+   comments from May 2009
+-  Rosie: The nested matches produced using the `Rosie pattern
+   language <https://rosie-lang.org/>`__ ``all.things`` pattern on a log
+   file
+-  Nytimes: Metadata of a collection of New York Times articles
+-  Tpch: The TPC-H database benchmark's deals table as a JSON file
+-  Twitter: A JSON export of the @pypyproject Twitter account data
+-  Wikidata: A file storing a subset of the Wikidata fact dump from Nov
+   11, 2014
+-  `Yelp <https://www.yelp.com/dataset/download>`__: A file of yelp
+   businesses
+
+Here are the file sizes of the benchmarks:
+
+.. raw:: html
+
+   <table border="1" class="dataframe">\n  <thead>\n    <tr style="text-align: right;">\n      <th>Benchmark</th>\n      <th>File Size [MiB]</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <td>Censys</td>\n      <td>898.45</td>\n    </tr>\n    <tr>\n      <td>Gharchive</td>\n      <td>276.34</td>\n    </tr>\n    <tr>\n      <td>NYTimes</td>\n      <td>12.98</td>\n    </tr>\n    <tr>\n      <td>Reddit</td>\n      <td>931.65</td>\n    </tr>\n    <tr>\n      <td>Rosie</td>\n      <td>388.88</td>\n    </tr>\n    <tr>\n      <td>TPCH</td>\n      <td>173.86</td>\n    </tr>\n    <tr>\n      <td>Wikidata</td>\n      <td>119.75</td>\n    </tr>\n    <tr>\n      <td>Yelp</td>\n      <td>167.61</td>\n    </tr>\n  </tbody>\n</table>
+
+
+I measured the times of each benchmark with a number of variations
+of the improved PyPy algorithms:
+
+-  PyPyBaseline: The PyPy JSON parser as it was before my work with JSON
+   parsing started (PyPy version 5.8)
+-  PyPyKeyStringCaching: Memoizing the key strings of dictionaries, but
+   not the other strings in a json file, and not using maps to represent
+   dictionaries (this is the JSON parser that PyPy has been shipping since
+   version 5.9, in the benchmarks I used 7.1).
+-  PyPyMapNoCache: Like PyPyKeyStringCaching, but using maps to
+   represent dictionaries. This includes speculatively parsing the next
+   key using memcmp, but does not use string caching of non-key strings.
+-  PyPyFull: Like PyPyMapNoCache but uses a string cache for all
+   strings, not just keys. This is equivalent to what is in PyPy 7.2
+
+In addition to wall clock time of parsing, I also measured the increase
+in memory use of each implementation after the input string has been
+deserialized, i.e. the size of the in-memory representation of every
+JSON file.
+
+
+Contributions of Individual Optimizations
+---------------------------------------------
+
+Let's first look at the contributions of the individual optimizations to the
+overall performance and memory usage.
+
+$$ two graphs
+
+All the benchmarks were run 30 times in new processes, all the numbers are
+normalized to PyPyFull.
+
+The biggest individual improvement to both parsing time and memory used comes
+from caching just the keys in parsed dictionaries. This is the optimization in
+PyPy's JSON parser that has been implemented for a while already. To understand
+why this optimization is so useful, let's look at some numbers about each
+benchmark, namely the number of total keys across all dictionaries in each
+file, as well as the number of unique keys. As we can see, for all benchmarks
+the number of unique keys is significantly smaller than the number of keys in
+total.
+
+.. raw:: html
+
+   <table border="1" class="dataframe">\n  <thead>\n    <tr style="text-align: right;">\n      <th>Benchmark</th>\n      <th>Number of keys</th>\n      <th>Number of unique keys</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <td>Censys</td>\n      <td>14\u2009404\u2009234</td>\n      <td>163</td>\n    </tr>\n    <tr>\n      <td>Gharchive</td>\n      <td>6\u2009637\u2009881</td>\n      <td>169</td>\n    </tr>\n    <tr>\n      <td>NYTimes</td>\n      <td>417\u2009337</td>\n      <td>60</td>\n    </tr>\n    <tr>\n      <td>Reddit</td>\n      <td>25\u2009226\u2009397</td>\n      <td>21</td>\n    </tr>\n    <tr>\n      <td>Rosie</td>\n      <td>28\u2009500\u2009101</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <td>TPCH</td>\n      <td>6\u2009700\u2009000</td>\n      <td>45</td>\n    </tr>\n    <tr>\n      <td>Wikidata</td>\n      <td>6\u2009235\u2009088</td>\n      <td>1\u2009602</td>\n    </tr>\n    <tr>\n      <td>Yelp</td>\n      <td>5\u2009133\u2009914</td>\n      <td>61</td>\n    </tr>\n  </tbody>\n</table>
+
+
+The next big jump in deserialization time and memory comes from introducing
+maps to represent deserialized dictionaries. With PyPyMapNoCache
+deserialization time goes down because it's much cheaper to walk the tree
+of maps and store all deserialized objects into an array of values than to
+build hashmaps with the same keys again and again. Memory use goes down
+for the same reason: it takes a lot less memory to store the shared
+structure of each set of keys in the map, as opposed to repeating it again
+and again in every hashmap.
+
+We can look at some numbers about every benchmark again. The table shows how
+many map-based dictionaries are deserialized for every benchmark, and how many
+hashmap-backed dictionaries. We see that the number of hashmap-backed
+dictionaries is often zero, or at most a small percentage of all dictionaries
+in each benchmark. Yelp has the biggest number of hashmap-backed dictionaries.
+The reason for this is that the input file contains hashmaps that store
+combinations of various features of Yelp businesses, and a lot of these
+combinations are totally unique to a business. Therefore the heuristics
+determine that it's better to store these using hashmaps.
+
+.. raw:: html
+
+   <table border="1" class="dataframe">\n  <thead>\n    <tr style="text-align: right;">\n      <th>Benchmark</th>\n      <th>Map Dicts</th>\n      <th>Regular Dicts</th>\n      <th>% Regular Dicts</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <td>Censys</td>\n      <td>4\u2009049\u2009235</td>\n      <td>1\u2009042</td>\n      <td>0.03</td>\n    </tr>\n    <tr>\n      <td>Gharchive</td>\n      <td>955\u2009301</td>\n      <td>0</td>\n      <td>0.00</td>\n    </tr>\n    <tr>\n      <td>NYTimes</td>\n      <td>80\u2009393</td>\n      <td>0</td>\n      <td>0.00</td>\n    </tr>\n    <tr>\n      <td>Reddit</td>\n      <td>1\u2009201\u2009257</td>\n      <td>0</td>\n      <td>0.00</td>\n    </tr>\n    <tr>\n      <td>Rosie</td>\n      <td>6\u2009248\u2009966</td>\n      <td>0</td>\n      <td>0.00</td>\n    </tr>\n    <tr>\n      <td>TPCH</td>\n      <td>1\u2009000\u2009000</td>\n      <td>0</td>\n      <td>0.00</td>\n    </tr>\n    <tr>\n      <td>Wikidata</td>\n      <td>1\u2009923\u2009460</td>\n      <td>46\u2009905</td>\n      <td>2.38</td>\n    </tr>\n    <tr>\n      <td>Yelp</td>\n      <td>443\u2009140</td>\n      <td>52\u2009051</td>\n      <td>10.51</td>\n    </tr>\n  </tbody>\n</table>
+
+
+We can also look at numbers about how often the memcmp-based speculative
+parsing of the next key of a given map succeeds. Looking at statistics
+about each benchmark, we can see that the speculation of what key we
+expect next pays off in a significant percentage of cases, between 63% for
+Wikidata where the dictionary structures are quite irregular, and 99% for
+Reddit, where all the dictionaries have the same set of keys.
+
+.. raw:: html
+
+   <table border="1" class="dataframe">\n  <thead>\n    <tr style="text-align: right;">\n      <th>Benchmark</th>\n      <th>Number of Keys</th>\n      <th>Map Transitions</th>\n      <th>% Successful Speculation</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <td>Censys</td>\n      <td>14\u2009404\u2009234</td>\n      <td>14\u2009403\u2009243</td>\n      <td>65.79</td>\n    </tr>\n    <tr>\n      <td>Gharchive</td>\n      <td>6\u2009637\u2009881</td>\n      <td>6\u2009637\u2009881</td>\n      <td>86.71</td>\n    </tr>\n    <tr>\n      <td>NYTimes</td>\n      <td>417\u2009337</td>\n      <td>417\u2009337</td>\n      <td>79.85</td>\n    </tr>\n    <tr>\n      <td>Reddit</td>\n      <td>25\u2009226\u2009397</td>\n      <td>25\u2009226\u2009397</td>\n      <td>100.00</td>\n    </tr>\n    <tr>\n      <td>Rosie</td>\n      <td>28\u2009500\u2009101</td>\n      <td>28\u2009500\u2009101</td>\n      <td>90.37</td>\n    </tr>\n    <tr>\n      <td>TPCH</td>\n      <td>6\u2009700\u2009000</td>\n      <td>6\u2009700\u2009000</td>\n      <td>86.57</td>\n    </tr>\n    <tr>\n      <td>Wikidata</td>\n      <td>6\u2009235\u2009088</td>\n      <td>5\u2009267\u2009744</td>\n      <td>63.68</td>\n    </tr>\n    <tr>\n      <td>Yelp</td>\n      <td>5\u2009133\u2009914</td>\n      <td>4\u2009593\u2009980</td>\n      <td>90.43</td>\n    </tr>\n    <tr>\n      <td>geomean</td>\n      <td></td>\n      <td></td>\n      <td>82.04</td>\n    </tr>\n  </tbody>\n</table>
+
+General string caching is the most unclear optimization. On the one hand its
+impact on memory usage is quite substantial, leading to a 20% reduction for
+Gharchive and Reddit, up to a 2&#215; improvement for Yelp. On the other hand, the
+effect on performance is less clear, since it even leads to a slowdown in
+Gharchive and Reddit, and generally only a small improvement. Choosing the
+right heuristic for when to disable the cache also has somewhat unclear effects
+and is definitely a topic worthy of further investigation.

From pypy.commits at gmail.com  Sun Oct  6 10:40:32 2019
From: pypy.commits at gmail.com (Yannick_Jadoul)
Date: Sun, 06 Oct 2019 07:40:32 -0700 (PDT)
Subject: [pypy-commit] pypy py3.7: Implemented __mro_entries__ from PEP 560
Message-ID: <5d99fce0.1c69fb81.9c2e1.c62d@mx.google.com>

Author: Yannick Jadoul <yannick.jadoul at belgacom.net>
Branch: py3.7
Changeset: r97729:02a99370f6b3
Date: 2019-09-24 17:27 +0200
http://bitbucket.org/pypy/pypy/changeset/02a99370f6b3/

Log:	Implemented __mro_entries__ from PEP 560

diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -96,13 +96,33 @@
     frame = space.getexecutioncontext().gettopframe()
     frame.exec_(w_prog, w_globals, w_locals)
 
+def _update_bases(space, w_bases):
+    bases_w = space.listview(w_bases)
+    new_bases_w = []
+    changed = False
+    for w_base in bases_w:
+        w_meth = space.lookup(w_base, '__mro_entries__')
+        if w_meth is not None:
+            new_base_w = space.get_and_call_function(w_meth, w_base, w_bases)
+            if not space.isinstance_w(new_base_w, space.w_tuple):
+                raise oefmt(space.w_TypeError, "__mro_entries__ must return a tuple")
+            new_bases_w.extend(space.fixedview(new_base_w))
+            changed = True
+        else:
+            new_bases_w.append(w_base)
+    if not changed:
+        return bases_w
+    return new_bases_w
+
 def build_class(space, w_func, w_name, __args__):
     from pypy.objspace.std.typeobject import _calculate_metaclass, W_TypeObject
     from pypy.interpreter.nestedscope import Cell
     if not isinstance(w_func, Function):
         raise oefmt(space.w_TypeError, "__build_class__: func must be a function")
-    bases_w, kwds_w = __args__.unpack()
-    w_bases = space.newtuple(bases_w)
+    orig_bases_w, kwds_w = __args__.unpack()
+    w_orig_bases = space.newtuple(orig_bases_w)
+    bases_w = _update_bases(space, w_orig_bases)
+    w_bases = space.newtuple(bases_w[:])
     w_meta = kwds_w.pop('metaclass', None)
     if w_meta is not None:
         isclass = space.isinstance_w(w_meta, space.w_type)
@@ -144,6 +164,8 @@
     frame = space.createframe(code, w_func.w_func_globals, w_func)
     frame.setdictscope(w_namespace)
     w_cell = frame.run()
+    if bases_w is not orig_bases_w:
+        space.setitem(w_namespace, space.newtext("__orig_bases__"), w_orig_bases)
     keywords = kwds_w.keys()
     args = Arguments(space,
                      args_w=[w_name, w_bases, w_namespace],
diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -1433,6 +1433,58 @@
         assert WithMetaclass[int] == "Metaclass[int]"
         """
 
+    def test_mro_entries(self):
+        """
+        class BaseA: pass
+        class BaseB: pass
+        class BaseC: pass
+        class BaseD: pass
+
+        class ProxyA:
+            def __mro_entries__(self, orig_bases):
+                return (BaseA,)
+        class ProxyAB:
+            def __mro_entries__(self, orig_bases):
+                return (BaseA, BaseB)
+        class ProxyNone:
+            def __mro_entries__(self, orig_bases):
+                return ()
+
+        class TestA(ProxyA()): pass
+        assert TestA.__bases__ == (BaseA,)
+        assert len(TestA.__orig_bases__) == 1
+        assert isinstance(TestA.__orig_bases__[0], ProxyA)
+
+        class TestAB(ProxyAB()): pass
+        assert TestAB.__bases__ == (BaseA, BaseB)
+        assert len(TestAB.__orig_bases__) == 1
+        assert isinstance(TestAB.__orig_bases__[0], ProxyAB)
+
+        class TestNone(ProxyNone()): pass
+        assert TestNone.__bases__ == (object,)
+        assert len(TestNone.__orig_bases__) == 1
+        assert isinstance(TestNone.__orig_bases__[0], ProxyNone)
+
+        class TestMixed(BaseC, ProxyAB(), BaseD, ProxyNone()): pass
+        assert TestMixed.__bases__ == (BaseC, BaseA, BaseB, BaseD)
+        assert len(TestMixed.__orig_bases__) == 4
+        assert isinstance(TestMixed.__orig_bases__[1], ProxyAB) and isinstance(TestMixed.__orig_bases__[3], ProxyNone)
+
+        with raises(TypeError) as excinfo:
+            class TestDuplicate(BaseB, ProxyAB()): pass
+        assert str(excinfo.value) == "duplicate base class 'BaseB'"
+
+        with raises(TypeError) as excinfo:
+            type('TestType', (BaseC, ProxyAB(), BaseD, ProxyNone()), {})
+        assert str(excinfo.value) == "type() doesn't support MRO entry resolution; use types.new_class()"
+
+        import types
+        TestTypesNewClass = types.new_class('TestTypesNewClass', (BaseC, ProxyAB(), BaseD, ProxyNone()), {})
+        assert TestMixed.__bases__ == (BaseC, BaseA, BaseB, BaseD)
+        assert len(TestMixed.__orig_bases__) == 4
+        assert isinstance(TestMixed.__orig_bases__[1], ProxyAB) and isinstance(TestMixed.__orig_bases__[3], ProxyNone)
+        """
+
 
 class AppTestWithMethodCacheCounter:
     spaceconfig = {"objspace.std.withmethodcachecounter": True}
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -808,6 +808,11 @@
     # above to be seen by the jit.
     _check_new_args(space, w_name, w_bases, w_dict)
     bases_w = space.fixedview(w_bases)
+    for w_base in bases_w:
+        if space.lookup(w_base, '__mro_entries__') is not None:
+            raise oefmt(space.w_TypeError,
+                        "type() doesn't support MRO entry resolution; "
+                        "use types.new_class()")
 
     w_winner = _calculate_metaclass(space, w_typetype, bases_w)
     if not space.is_w(w_winner, w_typetype):

From pypy.commits at gmail.com  Sun Oct  6 10:40:34 2019
From: pypy.commits at gmail.com (Yannick_Jadoul)
Date: Sun, 06 Oct 2019 07:40:34 -0700 (PDT)
Subject: [pypy-commit] pypy py3.7: Fixed annotation errors related to
 fixed-sized lists,
 and added a test checking __orig_bases__ is only set when __bases__ gets
 changed through the __mro_entries__ mechanism
Message-ID: <5d99fce2.1c69fb81.6ca67.597d@mx.google.com>

Author: Yannick Jadoul <yannick.jadoul at belgacom.net>
Branch: py3.7
Changeset: r97730:01b4c2af5928
Date: 2019-09-24 19:08 +0200
http://bitbucket.org/pypy/pypy/changeset/01b4c2af5928/

Log:	Fixed annotation errors related to fixed-sized lists, and added a
	test checking __orig_bases__ is only set when __bases__ gets changed
	through the __mro_entries__ mechanism

diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -97,7 +97,7 @@
     frame.exec_(w_prog, w_globals, w_locals)
 
 def _update_bases(space, w_bases):
-    bases_w = space.listview(w_bases)
+    bases_w = space.fixedview(w_bases)
     new_bases_w = []
     changed = False
     for w_base in bases_w:
@@ -112,7 +112,7 @@
             new_bases_w.append(w_base)
     if not changed:
         return bases_w
-    return new_bases_w
+    return new_bases_w[:]
 
 def build_class(space, w_func, w_name, __args__):
     from pypy.objspace.std.typeobject import _calculate_metaclass, W_TypeObject
@@ -122,7 +122,7 @@
     orig_bases_w, kwds_w = __args__.unpack()
     w_orig_bases = space.newtuple(orig_bases_w)
     bases_w = _update_bases(space, w_orig_bases)
-    w_bases = space.newtuple(bases_w[:])
+    w_bases = space.newtuple(bases_w)
     w_meta = kwds_w.pop('metaclass', None)
     if w_meta is not None:
         isclass = space.isinstance_w(w_meta, space.w_type)
diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -1483,6 +1483,10 @@
         assert TestMixed.__bases__ == (BaseC, BaseA, BaseB, BaseD)
         assert len(TestMixed.__orig_bases__) == 4
         assert isinstance(TestMixed.__orig_bases__[1], ProxyAB) and isinstance(TestMixed.__orig_bases__[3], ProxyNone)
+
+        class TestNoOrigBases(BaseA, BaseB): pass
+        assert TestNoOrigBases.__bases__ == (BaseA, BaseB)
+        assert not hasattr(TestNoOrigBases, '__orig_bases__')
         """
 
 

From pypy.commits at gmail.com  Sun Oct  6 10:40:36 2019
From: pypy.commits at gmail.com (Yannick_Jadoul)
Date: Sun, 06 Oct 2019 07:40:36 -0700 (PDT)
Subject: [pypy-commit] pypy py3.7: merge upstream py3.7
Message-ID: <5d99fce4.1c69fb81.667e4.7f98@mx.google.com>

Author: Yannick Jadoul <yannick.jadoul at belgacom.net>
Branch: py3.7
Changeset: r97731:0c9e1e22eaad
Date: 2019-10-03 01:18 +0200
http://bitbucket.org/pypy/pypy/changeset/0c9e1e22eaad/

Log:	merge upstream py3.7

diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -96,12 +96,32 @@
     frame = space.getexecutioncontext().gettopframe()
     frame.exec_(w_prog, w_globals, w_locals)
 
+def _update_bases(space, w_bases):
+    bases_w = space.fixedview(w_bases)
+    new_bases_w = []
+    changed = False
+    for w_base in bases_w:
+        w_meth = space.lookup(w_base, '__mro_entries__')
+        if w_meth is not None:
+            new_base_w = space.get_and_call_function(w_meth, w_base, w_bases)
+            if not space.isinstance_w(new_base_w, space.w_tuple):
+                raise oefmt(space.w_TypeError, "__mro_entries__ must return a tuple")
+            new_bases_w.extend(space.fixedview(new_base_w))
+            changed = True
+        else:
+            new_bases_w.append(w_base)
+    if not changed:
+        return bases_w
+    return new_bases_w[:]
+
 def build_class(space, w_func, w_name, __args__):
     from pypy.objspace.std.typeobject import _calculate_metaclass, W_TypeObject
     from pypy.interpreter.nestedscope import Cell
     if not isinstance(w_func, Function):
         raise oefmt(space.w_TypeError, "__build_class__: func must be a function")
-    bases_w, kwds_w = __args__.unpack()
+    orig_bases_w, kwds_w = __args__.unpack()
+    w_orig_bases = space.newtuple(orig_bases_w)
+    bases_w = _update_bases(space, w_orig_bases)
     w_bases = space.newtuple(bases_w)
     w_meta = kwds_w.pop('metaclass', None)
     if w_meta is not None:
@@ -144,6 +164,8 @@
     frame = space.createframe(code, w_func.w_func_globals, w_func)
     frame.setdictscope(w_namespace)
     w_cell = frame.run()
+    if bases_w is not orig_bases_w:
+        space.setitem(w_namespace, space.newtext("__orig_bases__"), w_orig_bases)
     keywords = kwds_w.keys()
     args = Arguments(space,
                      args_w=[w_name, w_bases, w_namespace],
diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -1433,6 +1433,62 @@
         assert WithMetaclass[int] == "Metaclass[int]"
         """
 
+    def test_mro_entries(self):
+        """
+        class BaseA: pass
+        class BaseB: pass
+        class BaseC: pass
+        class BaseD: pass
+
+        class ProxyA:
+            def __mro_entries__(self, orig_bases):
+                return (BaseA,)
+        class ProxyAB:
+            def __mro_entries__(self, orig_bases):
+                return (BaseA, BaseB)
+        class ProxyNone:
+            def __mro_entries__(self, orig_bases):
+                return ()
+
+        class TestA(ProxyA()): pass
+        assert TestA.__bases__ == (BaseA,)
+        assert len(TestA.__orig_bases__) == 1
+        assert isinstance(TestA.__orig_bases__[0], ProxyA)
+
+        class TestAB(ProxyAB()): pass
+        assert TestAB.__bases__ == (BaseA, BaseB)
+        assert len(TestAB.__orig_bases__) == 1
+        assert isinstance(TestAB.__orig_bases__[0], ProxyAB)
+
+        class TestNone(ProxyNone()): pass
+        assert TestNone.__bases__ == (object,)
+        assert len(TestNone.__orig_bases__) == 1
+        assert isinstance(TestNone.__orig_bases__[0], ProxyNone)
+
+        class TestMixed(BaseC, ProxyAB(), BaseD, ProxyNone()): pass
+        assert TestMixed.__bases__ == (BaseC, BaseA, BaseB, BaseD)
+        assert len(TestMixed.__orig_bases__) == 4
+        assert isinstance(TestMixed.__orig_bases__[1], ProxyAB) and isinstance(TestMixed.__orig_bases__[3], ProxyNone)
+
+        with raises(TypeError) as excinfo:
+            class TestDuplicate(BaseB, ProxyAB()): pass
+        assert str(excinfo.value) == "duplicate base class 'BaseB'"
+
+        with raises(TypeError) as excinfo:
+            type('TestType', (BaseC, ProxyAB(), BaseD, ProxyNone()), {})
+        assert str(excinfo.value) == "type() doesn't support MRO entry resolution; use types.new_class()"
+
+        import types
+        TestTypesNewClass = types.new_class('TestTypesNewClass', (BaseC, ProxyAB(), BaseD, ProxyNone()), {})
+        assert TestMixed.__bases__ == (BaseC, BaseA, BaseB, BaseD)
+        assert len(TestMixed.__orig_bases__) == 4
+        assert isinstance(TestMixed.__orig_bases__[1], ProxyAB) and isinstance(TestMixed.__orig_bases__[3], ProxyNone)
+
+        class TestNoOrigBases(BaseA, BaseB): pass
+        assert TestNoOrigBases.__bases__ == (BaseA, BaseB)
+        assert not hasattr(TestNoOrigBases, '__orig_bases__')
+        """
+
 
 class AppTestWithMethodCacheCounter:
     spaceconfig = {"objspace.std.withmethodcachecounter": True}
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -808,6 +808,11 @@
     # above to be seen by the jit.
     _check_new_args(space, w_name, w_bases, w_dict)
     bases_w = space.fixedview(w_bases)
+    for w_base in bases_w:
+        if space.lookup(w_base, '__mro_entries__') is not None:
+            raise oefmt(space.w_TypeError,
+                        "type() doesn't support MRO entry resolution; "
+                        "use types.new_class()")
 
     w_winner = _calculate_metaclass(space, w_typetype, bases_w)
     if not space.is_w(w_winner, w_typetype):

From pypy.commits at gmail.com  Sun Oct  6 11:06:34 2019
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 06 Oct 2019 08:06:34 -0700 (PDT)
Subject: [pypy-commit] pypy py3.7: Merged in
 Yannick_Jadoul/pypy/py3.7-pep562 (pull request #670)
Message-ID: <5d9a02fa.1c69fb81.6b8e2.023c@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.7
Changeset: r97733:dff15039dbab
Date: 2019-10-06 15:05 +0000
http://bitbucket.org/pypy/pypy/changeset/dff15039dbab/

Log:	Merged in Yannick_Jadoul/pypy/py3.7-pep562 (pull request #670)

	Implementation of PEP 562, adding __getattr__ and __dir__ for
	modules

diff --git a/pypy/interpreter/module.py b/pypy/interpreter/module.py
--- a/pypy/interpreter/module.py
+++ b/pypy/interpreter/module.py
@@ -134,6 +134,10 @@
         except OperationError as e:
             if not e.match(space, space.w_AttributeError):
                 raise
+            w_dict = self.w_dict
+            w_getattr = space.finditem(w_dict, space.newtext('__getattr__'))
+            if w_getattr is not None:
+                return space.call_function(w_getattr, w_attr)
             w_name = space.finditem(self.w_dict, space.newtext('__name__'))
             if w_name is None:
                 raise oefmt(space.w_AttributeError,
@@ -147,6 +151,9 @@
         if not space.isinstance_w(w_dict, space.w_dict):
             raise oefmt(space.w_TypeError, "%N.__dict__ is not a dictionary",
                         self)
+        w_dir = space.finditem(w_dict, space.newtext('__dir__'))
+        if w_dir is not None:
+            return space.call_function(w_dir)
         return space.call_function(space.w_list, w_dict)
 
     # These three methods are needed to implement '__class__' assignment
diff --git a/pypy/interpreter/test/test_module.py b/pypy/interpreter/test/test_module.py
--- a/pypy/interpreter/test/test_module.py
+++ b/pypy/interpreter/test/test_module.py
@@ -262,3 +262,29 @@
         raises(AttributeError, "del x.a")
         raises(TypeError, "x.__class__ = X")
         raises(TypeError, "sys.__class__ = XX")
+
+    def test_getattr_dir(self):
+        def __getattr__(name):
+            if name == 'y':
+                return 42
+            elif name == 'z':
+                return
+            raise AttributeError("No attribute '{}'".format(name))
+
+        def __dir__():
+            return ['x', 'y', 'z', 'w']
+
+        import sys
+        m = type(sys)('foo')
+        m.x = 'x'
+        m.__getattr__ = __getattr__
+        print(m.__dir__)
+        m.__dir__ = __dir__
+
+        assert m.x == 'x'
+        assert m.y == 42
+        assert m.z == None
+        excinfo = raises(AttributeError, 'm.w')
+        assert str(excinfo.value) == "No attribute 'w'"
+        print(dir(m))
+        assert dir(m) == sorted(['x', 'y', 'z', 'w'])

From pypy.commits at gmail.com  Sun Oct  6 11:06:42 2019
From: pypy.commits at gmail.com (Yannick_Jadoul)
Date: Sun, 06 Oct 2019 08:06:42 -0700 (PDT)
Subject: [pypy-commit] pypy py3.7-pep562: Implementation of PEP 562,
 adding __getattr__ and __dir__ for modules
Message-ID: <5d9a0302.1c69fb81.d03c0.879c@mx.google.com>

Author: Yannick Jadoul <yannick.jadoul at belgacom.net>
Branch: py3.7-pep562
Changeset: r97732:bb000a053f9f
Date: 2019-10-04 01:34 +0200
http://bitbucket.org/pypy/pypy/changeset/bb000a053f9f/

Log:	Implementation of PEP 562, adding __getattr__ and __dir__ for
	modules

diff --git a/pypy/interpreter/module.py b/pypy/interpreter/module.py
--- a/pypy/interpreter/module.py
+++ b/pypy/interpreter/module.py
@@ -134,6 +134,10 @@
         except OperationError as e:
             if not e.match(space, space.w_AttributeError):
                 raise
+            w_dict = self.w_dict
+            w_getattr = space.finditem(w_dict, space.newtext('__getattr__'))
+            if w_getattr is not None:
+                return space.call_function(w_getattr, w_attr)
             w_name = space.finditem(self.w_dict, space.newtext('__name__'))
             if w_name is None:
                 raise oefmt(space.w_AttributeError,
@@ -147,6 +151,9 @@
         if not space.isinstance_w(w_dict, space.w_dict):
             raise oefmt(space.w_TypeError, "%N.__dict__ is not a dictionary",
                         self)
+        w_dir = space.finditem(w_dict, space.newtext('__dir__'))
+        if w_dir is not None:
+            return space.call_function(w_dir)
         return space.call_function(space.w_list, w_dict)
 
     # These three methods are needed to implement '__class__' assignment
diff --git a/pypy/interpreter/test/test_module.py b/pypy/interpreter/test/test_module.py
--- a/pypy/interpreter/test/test_module.py
+++ b/pypy/interpreter/test/test_module.py
@@ -262,3 +262,29 @@
         raises(AttributeError, "del x.a")
         raises(TypeError, "x.__class__ = X")
         raises(TypeError, "sys.__class__ = XX")
+
+    def test_getattr_dir(self):
+        def __getattr__(name):
+            if name == 'y':
+                return 42
+            elif name == 'z':
+                return
+            raise AttributeError("No attribute '{}'".format(name))
+
+        def __dir__():
+            return ['x', 'y', 'z', 'w']
+
+        import sys
+        m = type(sys)('foo')
+        m.x = 'x'
+        m.__getattr__ = __getattr__
+        print(m.__dir__)
+        m.__dir__ = __dir__
+
+        assert m.x == 'x'
+        assert m.y == 42
+        assert m.z == None
+        excinfo = raises(AttributeError, 'm.w')
+        assert str(excinfo.value) == "No attribute 'w'"
+        print(dir(m))
+        assert dir(m) == sorted(['x', 'y', 'z', 'w'])

From pypy.commits at gmail.com  Sun Oct  6 15:11:47 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 06 Oct 2019 12:11:47 -0700 (PDT)
Subject: [pypy-commit] pypy _siphash24-collecting: try to align the starts
 differently and pass translation on aarch64
Message-ID: <5d9a3c73.1c69fb81.42918.27fd@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: _siphash24-collecting
Changeset: r97734:1fdf07c8f10e
Date: 2019-10-06 22:10 +0300
http://bitbucket.org/pypy/pypy/changeset/1fdf07c8f10e/

Log:	try to align the starts differently and pass translation on aarch64

diff --git a/rpython/rlib/rsiphash.py b/rpython/rlib/rsiphash.py
--- a/rpython/rlib/rsiphash.py
+++ b/rpython/rlib/rsiphash.py
@@ -256,8 +256,9 @@
     v2 = k0 ^ magic2
     v3 = k1 ^ magic3
 
-    direct = (SZ == 1) and (misaligned_is_fine or
-                 (rffi.cast(lltype.Signed, addr_in) & 7) == 0)
+    # direct = (SZ == 1) and (misaligned_is_fine or
+    #             (rffi.cast(lltype.Signed, addr_in) & 7) == 0)
+    direct = (SZ == 1) and misaligned_is_fine
     if direct:
         assert SZ == 1
         while size >= 8:

From pypy.commits at gmail.com  Mon Oct  7 14:32:06 2019
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 07 Oct 2019 11:32:06 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6-asyncgen: Remove unused argument
 in_generator from execute_frame()
Message-ID: <5d9b84a6.1c69fb81.af29f.4705@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6-asyncgen
Changeset: r97735:43827ffdbe61
Date: 2019-10-07 19:31 +0100
http://bitbucket.org/pypy/pypy/changeset/43827ffdbe61/

Log:	Remove unused argument in_generator from execute_frame()

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -125,7 +125,7 @@
                             self.KIND)
         self.running = True
         try:
-            w_result = frame.execute_frame(self, w_arg_or_err)
+            w_result = frame.execute_frame(w_arg_or_err)
         except OperationError as e:
             # errors finish a frame
             try:
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -351,9 +351,9 @@
         else:
             return r_uint(0)
 
-    def execute_frame(self, in_generator=None, w_arg_or_err=None):
+    def execute_frame(self, w_arg_or_err=None):
         """Execute this frame.  Main entry point to the interpreter.
-        'in_generator' is non-None iff we are starting or resuming
+        'w_arg_or_err' is non-None iff we are starting or resuming
         a generator or coroutine frame; in that case, w_arg_or_err
         is the input argument -or- an SApplicationException instance.
         """
@@ -375,7 +375,7 @@
             # the YIELD_VALUE/YIELD_FROM instruction.
             try:
                 try:
-                    if in_generator is None:
+                    if w_arg_or_err is None:
                         assert self.last_instr == -1
                         next_instr = r_uint(0)
                     else:
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1151,8 +1151,8 @@
     def YIELD_FROM(self, oparg, next_instr):
         # Unlike CPython, we handle this not by repeating the same
         # bytecode over and over until the inner iterator is exhausted.
-        # Instead, we directly set the generator's w_yielded_from.
-        # This asks generator.resume_execute_frame() to exhaust that
+        # Instead, we set w_yielding_from.
+        # This asks resume_execute_frame() to exhaust that
         # sub-iterable first before continuing on the next bytecode.
         w_inputvalue = self.popvalue()    # that's always w_None, actually
         w_gen = self.popvalue()

From pypy.commits at gmail.com  Mon Oct  7 15:53:37 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 07 Oct 2019 12:53:37 -0700 (PDT)
Subject: [pypy-commit] pypy default: add jit help to documentation
Message-ID: <5d9b97c1.1c69fb81.8c9a.6c8b@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r97736:42337a484364
Date: 2019-10-07 22:51 +0300
http://bitbucket.org/pypy/pypy/changeset/42337a484364/

Log:	add jit help to documentation

diff --git a/pypy/doc/commandline_ref.rst b/pypy/doc/commandline_ref.rst
--- a/pypy/doc/commandline_ref.rst
+++ b/pypy/doc/commandline_ref.rst
@@ -9,3 +9,4 @@
 
    man/pypy.1.rst
    man/pypy3.1.rst
+   jit_help.rst
diff --git a/pypy/doc/jit_help.rst b/pypy/doc/jit_help.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/jit_help.rst
@@ -0,0 +1,78 @@
+========
+JIT help
+========
+
+.. note this is from ``pypy --jit help``
+
+Advanced JIT options
+====================
+
+``<pypy> --jit`` [*options*] where *options* is a comma-separated list of
+``OPTION=VALUE``:
+
+ decay=N
+    amount to regularly decay counters by (0=none, 1000=max) (default 40)
+
+ disable_unrolling=N
+    after how many operations we should not unroll (default 200)
+
+ enable_opts=N
+    INTERNAL USE ONLY (MAY NOT WORK OR LEAD TO CRASHES): optimizations to
+    enable, or all =
+    intbounds:rewrite:virtualize:string:pure:earlyforce:heap:unroll (default
+    all)
+
+ function_threshold=N
+    number of times a function must run for it to become traced from start
+    (default 1619)
+
+ inlining=N
+    inline python functions or not (1/0) (default 1)
+
+ loop_longevity=N
+    a parameter controlling how long loops will be kept before being freed,
+    an estimate (default 1000)
+
+ max_retrace_guards=N
+    number of extra guards a retrace can cause (default 15)
+
+ max_unroll_loops=N
+    number of extra unrollings a loop can cause (default 0)
+
+ max_unroll_recursion=N
+    how many levels deep to unroll a recursive function (default 7)
+
+ retrace_limit=N
+    how many times we can try retracing before giving up (default 0)
+
+ threshold=N
+    number of times a loop has to run for it to become hot (default 1039)
+
+ trace_eagerness=N
+    number of times a guard has to fail before we start compiling a bridge
+    (default 200)
+
+ trace_limit=N
+    number of recorded operations before we abort tracing with ABORT_TOO_LONG
+    (default 6000)
+
+ vec=N
+    turn on the vectorization optimization (vecopt). Supports x86 (SSE 4.1),
+    powerpc (SVX), s390x SIMD (default 0)
+
+ vec_all=N
+    try to vectorize trace loops that occur outside of the numpypy library
+    (default 0)
+
+ vec_cost=N
+    threshold for which traces to bail. Unpacking increases the counter,
+    vector operation decrease the cost (default 0)
+
+ off
+    turn off the JIT
+ help
+    print this page
+
+The :ref:`pypyjit<jit-hooks>` module can be used to control the JIT from inside
+pypy
+

From pypy.commits at gmail.com  Mon Oct  7 15:53:39 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 07 Oct 2019 12:53:39 -0700 (PDT)
Subject: [pypy-commit] pypy default: clean up "make html" warnings,
 remove a few blank whatsnew files
Message-ID: <5d9b97c3.1c69fb81.3eb93.2f9d@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r97737:96a1c0a30e47
Date: 2019-10-07 22:52 +0300
http://bitbucket.org/pypy/pypy/changeset/96a1c0a30e47/

Log:	clean up "make html" warnings, remove a few blank whatsnew files

diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -42,7 +42,8 @@
 -------------------------------
 
 .. toctree::
-   whatsnew-pypy3-head.rst
+    whatsnew-pypy3-head.rst
+    whatsnew-pypy3-7.1.0.rst
 
 CPython 3.5 compatible versions
 -------------------------------
@@ -50,6 +51,8 @@
 .. toctree::
 
    whatsnew-pypy3-7.0.0.rst
+   whatsnew-pypy3-6.0.0.rst
+   whatsnew-pypy3-5.10.0.rst
    whatsnew-pypy3-5.9.0.rst
    whatsnew-pypy3-5.8.0.rst
    whatsnew-pypy3-5.7.0.rst
@@ -62,10 +65,4 @@
    whatsnew-pypy3-5.5.0.rst
    whatsnew-pypy3-5.1.1-alpha1.rst
 
-CPython 3.2 compatible versions
--------------------------------
 
-.. toctree::
-
-   whatsnew-pypy3-2.4.0.rst
-   whatsnew-pypy3-2.3.1.rst
diff --git a/pypy/doc/release-v7.2.0.rst b/pypy/doc/release-v7.2.0.rst
--- a/pypy/doc/release-v7.2.0.rst
+++ b/pypy/doc/release-v7.2.0.rst
@@ -307,6 +307,7 @@
 .. _33786 : https://bugs.python.org/issue33786
 .. _32270 : https://bugs.python.org/issue32270
 .. _28691 : https://bugs.python.org/issue28691
+.. _33729 : https://bugs.python.org/issue33729
 
 .. _opencv2: https://github.com/skvark/opencv-python/
 .. _`issue 2617`: https://bitbucket.com/pypy/pypy/issues/2617
diff --git a/pypy/doc/whatsnew-pypy2-5.7.0.rst b/pypy/doc/whatsnew-pypy2-5.7.0.rst
--- a/pypy/doc/whatsnew-pypy2-5.7.0.rst
+++ b/pypy/doc/whatsnew-pypy2-5.7.0.rst
@@ -14,6 +14,7 @@
 changed in exactly the same way because of this fix.
 
 
+
 .. branch: rpython-error-to-systemerror
 
 Any uncaught RPython exception (from a PyPy bug) is turned into an
diff --git a/pypy/doc/whatsnew-pypy3-2.3.1.rst b/pypy/doc/whatsnew-pypy3-2.3.1.rst
deleted file mode 100644
--- a/pypy/doc/whatsnew-pypy3-2.3.1.rst
+++ /dev/null
@@ -1,6 +0,0 @@
-=========================
-What's new in PyPy3 2.3.1
-=========================
-
-.. this is a revision shortly after pypy3-release-2.3.x
-.. startrev: 0137d8e6657d
diff --git a/pypy/doc/whatsnew-pypy3-2.4.0.rst b/pypy/doc/whatsnew-pypy3-2.4.0.rst
deleted file mode 100644
--- a/pypy/doc/whatsnew-pypy3-2.4.0.rst
+++ /dev/null
@@ -1,6 +0,0 @@
-=========================
-What's new in PyPy3 2.4.0
-=========================
-
-.. this is a revision shortly after pypy3-release-2.4.x
-.. startrev: 12b940544622

From pypy.commits at gmail.com  Mon Oct  7 17:42:45 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 07 Oct 2019 14:42:45 -0700 (PDT)
Subject: [pypy-commit] pypy _siphash24-collecting: simplify test but it
 still passes (it should fail...)
Message-ID: <5d9bb155.1c69fb81.f054b.a264@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: _siphash24-collecting
Changeset: r97739:158a54d11ca5
Date: 2019-10-08 00:41 +0300
http://bitbucket.org/pypy/pypy/changeset/158a54d11ca5/

Log:	simplify test but it still passes (it should fail...)

diff --git a/rpython/memory/gctransform/test/test_framework.py b/rpython/memory/gctransform/test/test_framework.py
--- a/rpython/memory/gctransform/test/test_framework.py
+++ b/rpython/memory/gctransform/test/test_framework.py
@@ -151,101 +151,31 @@
 def test_cast_no_collect():
     # mimic some of the behaviour of _siphash24
     from rpython.rlib import rgc
-    from rpython.rlib.rarithmetic import r_uint64, r_uint32
+    from rpython.rlib.rarithmetic import r_uint64
     from rpython.translator.c.genc import CStandaloneBuilder
 
     @always_inline
-    def _rotate(x, b):
-        return (x << b) | (x >> (64 - b))
-
-    @always_inline
-    def _half_round(a, b, c, d, s, t):
-        a += b
-        c += d
-        b = _rotate(b, s) ^ a
-        d = _rotate(d, t) ^ c
-        a = _rotate(a, 32)
-        return a, b, c, d
-
-    @always_inline
     def _double_round(v0, v1, v2, v3):
-        v0,v1,v2,v3 = _half_round(v0,v1,v2,v3,13,16)
-        v2,v1,v0,v3 = _half_round(v2,v1,v0,v3,17,21)
-        v0,v1,v2,v3 = _half_round(v0,v1,v2,v3,13,16)
-        v2,v1,v0,v3 = _half_round(v2,v1,v0,v3,17,21)
         return v0, v1, v2, v3
 
     @rgc.no_collect
-    def _siphash24(addr_in, size, SZ=1):
-        index = 0
+    def _siphash24(addr_in):
         v0 =  r_uint64(0x736f6d6570736575)
         v1 =  r_uint64(0x646f72616e646f6d)
         v2 =  r_uint64(0x6c7967656e657261)
         v3 =  r_uint64(0x7465646279746573)
 
-        b = r_uint64(size) << 56
-        while size >= 8:
-            mi = (
-              r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index)) |
-              r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 1*SZ)) << 8 |
-              r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 2*SZ)) << 16 |
-              r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 3*SZ)) << 24 |
-              r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 4*SZ)) << 32 |
-              r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 5*SZ)) << 40 |
-              r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 6*SZ)) << 48 |
-              r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 7*SZ)) << 56
-            )
-            size -= 8
-            index += 8*SZ
-            v3 ^= mi
-            v0, v1, v2, v3 = _double_round(v0, v1, v2, v3)
-            v0 ^= mi
-
-        t = r_uint64(0)
-        if size == 7:
-            t = r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 6*SZ)) << 48
-            size = 6
-        if size == 6:
-            t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 5*SZ)) << 40
-            size = 5
-        if size == 5:
-            t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 4*SZ)) << 32
-            size = 4
-        if size == 4:
-            t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 3*SZ))<< 24
-            size = 3
-        if size == 3:
-            t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 2*SZ)) << 16
-            size = 2
-        if size == 2:
-            t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 1*SZ)) << 8
-            size = 1
-        if size == 1:
-            t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index))
-            size = 0
-        assert size == 0
-
-        b |= t
-
-        v3 ^= b
+        mi = r_uint64(0)
+        v3 ^= mi
         v0, v1, v2, v3 = _double_round(v0, v1, v2, v3)
-        v0 ^= b
-        v2 ^= 0xff
-        v0, v1, v2, v3 = _double_round(v0, v1, v2, v3)
-        v0, v1, v2, v3 = _double_round(v0, v1, v2, v3)
+        v0 ^= mi
 
         return (v0 ^ v1) ^ (v2 ^ v3)
 
-    def siphash24(s):
-        """'s' is a normal string.  Returns its siphash-2-4 as a r_uint64.
-        Don't forget to cast the result to a regular integer if needed,
-        e.g. with rarithmetic.intmask().
-        """
-        with rffi.scoped_nonmovingbuffer(s) as p:
-            return _siphash24(llmemory.cast_ptr_to_adr(p), len(s))
+
 
     def entrypoint(argv):
-        return siphash24('abc')
+        return _siphash24('abc')
 
     t = rtype(entrypoint, [s_list_of_strings])
     t.config.translation.gc = "minimark"

From pypy.commits at gmail.com  Wed Oct  9 03:38:41 2019
From: pypy.commits at gmail.com (stevie_92)
Date: Wed, 09 Oct 2019 00:38:41 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-gc-cycle: Implemented configurable limit
 for incremental rrc collection
Message-ID: <5d9d8e81.1c69fb81.dd898.d03f@mx.google.com>

Author: Stefan Beyer <home at sbeyer.at>
Branch: cpyext-gc-cycle
Changeset: r97741:518eb87d054c
Date: 2019-10-09 09:37 +0200
http://bitbucket.org/pypy/pypy/changeset/518eb87d054c/

Log:	Implemented configurable limit for incremental rrc collection
	Implemented working set to improve overall performance

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -2415,7 +2415,6 @@
             # finalizers/weak references are rare and short which means that
             # they do not need a separate state and do not need to be
             # made incremental.
-            # For now, the same applies to rawrefcount'ed objects.
             if rrc_finished:
                 ll_assert(not (self.probably_young_objects_with_finalizers
                                .non_empty()),
diff --git a/rpython/memory/gc/rrc/base.py b/rpython/memory/gc/rrc/base.py
--- a/rpython/memory/gc/rrc/base.py
+++ b/rpython/memory/gc/rrc/base.py
@@ -1,6 +1,7 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory, llgroup, rffi
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop
+from rpython.memory.gc import env
 
 def choose_rrc_gc_from_config(config):
     if config.translation.rrcgc:
@@ -118,6 +119,11 @@
         self.tuple_maybe_untrack = tuple_maybe_untrack
         self.state = self.STATE_DEFAULT
         self.cycle_enabled = True
+        inc_limit = env.read_uint_from_env('PYPY_RRC_GC_INCREMENT_STEP')
+        if inc_limit > 0:
+            self.inc_limit = inc_limit
+        else:
+            self.inc_limit = 1000
 
     def create_link_pypy(self, gcobj, pyobject):
         obj = llmemory.cast_ptr_to_adr(gcobj)
@@ -566,8 +572,14 @@
         pygchdr = self.pyobj_as_gc(pyobj)
         if pygchdr <> lltype.nullptr(self.PYOBJ_GC_HDR):
             if pygchdr.c_gc_refs != self.RAWREFCOUNT_REFS_UNTRACKED:
-                pygchdr.c_gc_refs += self.refcnt_add << \
-                                     self.RAWREFCOUNT_REFS_SHIFT
+                if (self.state != self.STATE_GARBAGE_MARKING and
+                        pygchdr.c_gc_refs >> self.RAWREFCOUNT_REFS_SHIFT == 0
+                        and self.refcnt_add > 0):
+                    addr = llmemory.cast_ptr_to_adr(pygchdr)
+                    self.pyobj_to_trace.append(addr)
+                else:
+                    pygchdr.c_gc_refs += (self.refcnt_add <<
+                                          self.RAWREFCOUNT_REFS_SHIFT)
         elif pyobj.c_ob_pypy_link != 0:
             pyobj.c_ob_refcnt += self.refcnt_add
             if self.refcnt_add > 0:
diff --git a/rpython/memory/gc/rrc/incmark.py b/rpython/memory/gc/rrc/incmark.py
--- a/rpython/memory/gc/rrc/incmark.py
+++ b/rpython/memory/gc/rrc/incmark.py
@@ -16,37 +16,46 @@
         if self.state == self.STATE_DEFAULT:
             # untrack all tuples with only non-gc rrc objects and
             # promote all other tuples to the pyobj_list
-            self._untrack_tuples()
-            # TODO: execute incrementally? (before snapshot!, own phase)
+            self._untrack_tuples() # execute incrementally?
 
             # now take a snapshot
             self._take_snapshot()
             self._debug_print_snap(print_label="after-snapshot")
 
-            # collect all rawrefcounted roots
-            self._collect_roots()
-            # TODO: execute incrementally (own phase, save index)
-
-            self._debug_print_snap(print_label="roots-marked")
-            self._debug_check_consistency(print_label="roots-marked")
-
-            self._gc_list_init(self.pyobj_old_list)
             self.state = self.STATE_MARKING
+            self.marking_state = 0
             return False
 
         if self.state == self.STATE_MARKING:
-            # mark all objects reachable from rawrefcounted roots
-            all_rrc_marked = self._mark_rawrefcount()
-            # TODO: execute incrementally
-
-            if (all_rrc_marked and not self.gc.objects_to_trace.non_empty() and
-                    not self.gc.more_objects_to_trace.non_empty()):
-                # all objects have been marked, dead objects will stay dead
-                self._debug_print_snap(print_label="before-fin")
-                self._debug_check_consistency(print_label="before-fin")
-                self.state = self.STATE_GARBAGE_MARKING
+            if self.marking_state == 0:
+                # collect all rawrefcounted roots
+                self._collect_roots() # execute incrementally (save index)?
+                self._debug_print_snap(print_label="roots-marked")
+                self._debug_check_consistency(print_label="roots-marked")
+                self._gc_list_init(self.pyobj_old_list)
+                self.marking_state = 1
+                return False
+            elif self.marking_state == 1:
+                # initialize working set from roots, then pause
+                self.pyobj_to_trace = self.gc.AddressStack()
+                for i in range(0, self.total_objs):
+                    obj = self.snapshot_objs[i]
+                    self._mark_rawrefcount_obj(obj)
+                self.p_list_old.foreach(self._mark_rawrefcount_linked, None)
+                self.o_list_old.foreach(self._mark_rawrefcount_linked, None)
+                self.marking_state = 2
+                return False
             else:
-                return False
+                # mark all objects reachable from rawrefcounted roots
+                all_rrc_marked = self._mark_rawrefcount()
+                if (all_rrc_marked and not self.gc.objects_to_trace.non_empty() and
+                        not self.gc.more_objects_to_trace.non_empty()):
+                    # all objects have been marked, dead objects will stay dead
+                    self._debug_print_snap(print_label="before-fin")
+                    self._debug_check_consistency(print_label="before-fin")
+                    self.state = self.STATE_GARBAGE_MARKING
+                else:
+                    return False
 
         # we are finished with marking, now finish things up
         ll_assert(self.state == self.STATE_GARBAGE_MARKING, "invalid state")
@@ -91,7 +100,8 @@
 
         # sync p_list_old (except gc-objects)
         # simply iterate the snapshot for objects in p_list, as linked objects
-        # might not be freed, except by the gc
+        # might not be freed, except by the gc; p_list is always at the
+        # beginning of the snapshot, so break if we reached a different pyobj
         free_p_list = self.gc.AddressStack()
         for i in range(0, self.total_objs):
             snapobj = self.snapshot_objs[i]
@@ -102,6 +112,10 @@
             if (pygchdr != lltype.nullptr(self.PYOBJ_GC_HDR) and
                     pygchdr.c_gc_refs != self.RAWREFCOUNT_REFS_UNTRACKED):
                 break  # only look for non-gc
+            addr = llmemory.cast_int_to_adr(snapobj.pypy_link)
+            if (self.gc.header(addr).tid &
+                    (self.GCFLAG_VISITED | self.GCFLAG_NO_HEAP_PTRS)):
+                continue # keep proxy if obj is marked
             if snapobj.refcnt == 0:
                 # check consistency
                 consistent = pyobj.c_ob_refcnt == snapobj.refcnt_original
@@ -269,17 +283,23 @@
         # objects are found, increment the refcount of all referenced objects
         # of those newly found objects
         reached_limit = False
-        found_alive = True
         simple_limit = 0
+        first = True # rescan proxies, in case only non-rc have been marked
         #
-        while found_alive and not reached_limit: # TODO: working set to improve performance?
-            found_alive = False
-            for i in range(0, self.total_objs):
-                obj = self.snapshot_objs[i]
-                found_alive |= self._mark_rawrefcount_obj(obj)
-            simple_limit += 1
-            if simple_limit > 3: # TODO: implement sane limit
-                reached_limit
+        while first or (self.pyobj_to_trace.non_empty() and not reached_limit):
+            while self.pyobj_to_trace.non_empty() and not reached_limit:
+                addr = self.pyobj_to_trace.pop()
+                snapobj = llmemory.cast_adr_to_ptr(addr,
+                                                   self.PYOBJ_SNAPSHOT_OBJ_PTR)
+                snapobj.refcnt += 1
+                self._mark_rawrefcount_obj(snapobj)
+                simple_limit += 1
+                if simple_limit > self.inc_limit: # TODO: add test
+                    reached_limit = True
+
+            self.p_list_old.foreach(self._mark_rawrefcount_linked, None)
+            self.o_list_old.foreach(self._mark_rawrefcount_linked, None)
+            first = False
         return not reached_limit # are there any objects left?
 
     def _mark_rawrefcount_obj(self, snapobj):
@@ -301,17 +321,38 @@
                 obj_ref = llmemory.cast_adr_to_ptr(addr,
                                                    self.PYOBJ_SNAPSHOT_OBJ_PTR)
                 if obj_ref != lltype.nullptr(self.PYOBJ_SNAPSHOT_OBJ):
-                    obj_ref.refcnt += 1
+                    if obj_ref.refcnt == 0:
+                        addr = llmemory.cast_ptr_to_adr(obj_ref)
+                        self.pyobj_to_trace.append(addr)
+                    else:
+                        obj_ref.refcnt += 1
             # mark recursively, if it is a pypyobj
             if snapobj.pypy_link <> 0:
                 intobj = snapobj.pypy_link
                 obj = llmemory.cast_int_to_adr(intobj)
                 self.gc.objects_to_trace.append(obj)
-                self.gc.visit_all_objects()  # TODO: remove to improve pause times
+                self.gc.visit_all_objects()  # TODO: move to outer loop, implement sane limit (ex. half of normal limit), retrace proxies
             # mark as processed
             snapobj.status = 0
         return alive
 
+    def _mark_rawrefcount_linked(self, pyobject, ignore):
+        # we only have to take gc-objs into consideration, rc-proxies only
+        # keep their non-rc objs alive (see _major_free)
+        pyobj = self._pyobj(pyobject)
+        addr = llmemory.cast_int_to_adr(pyobj.c_ob_pypy_link)
+        if self.gc.header(addr).tid & (self.GCFLAG_VISITED |
+                                       self.GCFLAG_NO_HEAP_PTRS):
+            pygchdr = self.pyobj_as_gc(pyobj)
+            if (pygchdr != lltype.nullptr(self.PYOBJ_GC_HDR) and
+                    pygchdr.c_gc_refs > 0 and
+                    pygchdr.c_gc_refs != self.RAWREFCOUNT_REFS_UNTRACKED):
+                index = pygchdr.c_gc_refs - 1
+                snapobj = self.snapshot_objs[index]
+                if snapobj.refcnt == 0:
+                    addr = llmemory.cast_ptr_to_adr(snapobj)
+                    self.pyobj_to_trace.append(addr)
+
     def _take_snapshot(self):
         total_refcnt = 0
         total_objs = 0
diff --git a/rpython/memory/gc/rrc/mark.py b/rpython/memory/gc/rrc/mark.py
--- a/rpython/memory/gc/rrc/mark.py
+++ b/rpython/memory/gc/rrc/mark.py
@@ -166,25 +166,36 @@
             self._gc_list_init(self.pyobj_old_list)
         else:
             self._gc_list_move(self.pyobj_list, self.pyobj_old_list)
+        pyobj_old = self.pyobj_list
+
+        # initialize working set
+        self.pyobj_to_trace = self.gc.AddressStack()
+        gchdr = self.pyobj_old_list.c_gc_next
+        while gchdr <> self.pyobj_old_list:
+            next_old = gchdr.c_gc_next
+            self._mark_rawrefcount_obj(gchdr, pyobj_old)
+            gchdr = next_old
+        gchdr = self.pyobj_isolate_old_list.c_gc_next
+        while gchdr <> self.pyobj_isolate_old_list:
+            next_old = gchdr.c_gc_next
+            self._mark_rawrefcount_obj(gchdr, pyobj_old)
+            gchdr = next_old
+        self.p_list_old.foreach(self._mark_rawrefcount_linked, None)
+        self.o_list_old.foreach(self._mark_rawrefcount_linked, None)
+
         # as long as new objects with cyclic a refcount > 0 or alive border
         # objects are found, increment the refcount of all referenced objects
         # of those newly found objects
-        found_alive = True
-        pyobj_old = self.pyobj_list
-        #
-        while found_alive: # TODO: working set to improve performance?
-            found_alive = False
-            gchdr = self.pyobj_old_list.c_gc_next
-            while gchdr <> self.pyobj_old_list:
-                next_old = gchdr.c_gc_next
-                found_alive |= self._mark_rawrefcount_obj(gchdr, pyobj_old)
-                gchdr = next_old
-            gchdr = self.pyobj_isolate_old_list.c_gc_next
-            while gchdr <> self.pyobj_isolate_old_list:
-                next_old = gchdr.c_gc_next
-                found_alive |= self._mark_rawrefcount_obj(gchdr, pyobj_old)
-                gchdr = next_old
-        #
+        while self.pyobj_to_trace.non_empty():
+            while self.pyobj_to_trace.non_empty():
+                addr = self.pyobj_to_trace.pop()
+                gchdr = llmemory.cast_adr_to_ptr(addr, self.PYOBJ_GC_HDR_PTR)
+                gchdr.c_gc_refs += 1 << self.RAWREFCOUNT_REFS_SHIFT
+                self._mark_rawrefcount_obj(gchdr, pyobj_old)
+            self.gc.visit_all_objects()
+            self.p_list_old.foreach(self._mark_rawrefcount_linked, None)
+            self.o_list_old.foreach(self._mark_rawrefcount_linked, None)
+
         # now all rawrefcounted objects, which are alive, have a cyclic
         # refcount > 0 or are marked
 
@@ -212,5 +223,15 @@
             # mark recursively, if it is a pypyobj
             if pyobj.c_ob_pypy_link <> 0:
                 self.gc.objects_to_trace.append(obj)
-                self.gc.visit_all_objects()
         return alive
+
+    def _mark_rawrefcount_linked(self, pyobject, ignore):
+        pyobj = self._pyobj(pyobject)
+        obj = self.refcnt_dict.get(pyobject)
+        if self.gc.header(obj).tid & (self.GCFLAG_VISITED |
+                                      self.GCFLAG_NO_HEAP_PTRS):
+            gchdr = self.pyobj_as_gc(pyobj)
+            if gchdr <> lltype.nullptr(self.PYOBJ_GC_HDR):
+                if gchdr.c_gc_refs >> self.RAWREFCOUNT_REFS_SHIFT == 0:
+                    addr = llmemory.cast_ptr_to_adr(gchdr)
+                    self.pyobj_to_trace.append(addr)
diff --git a/rpython/memory/gc/test/test_rawrefcount.py b/rpython/memory/gc/test/test_rawrefcount.py
--- a/rpython/memory/gc/test/test_rawrefcount.py
+++ b/rpython/memory/gc/test/test_rawrefcount.py
@@ -104,6 +104,7 @@
                                  rawrefcount_pyobj_as_gc,
                                  rawrefcount_finalizer_type,
                                  rawrefcount_tuple_maybe_untrack)
+        self.gc.rrc_gc.inc_limit = 2 # low limit to test incremental collection
 
     def _collect(self, major, expected_trigger=0):
         if major:

From pypy.commits at gmail.com  Wed Oct  9 05:06:49 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 09 Oct 2019 02:06:49 -0700 (PDT)
Subject: [pypy-commit] pypy _siphash24-collecting: be closer to what the
 translation does, which actually makes the test pass
Message-ID: <5d9da329.1c69fb81.197af.749a@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: _siphash24-collecting
Changeset: r97742:3a1a3aada546
Date: 2019-10-09 11:06 +0200
http://bitbucket.org/pypy/pypy/changeset/3a1a3aada546/

Log:	be closer to what the translation does, which actually makes the
	test pass (hence it didn't reproduce the actual problem)

diff --git a/rpython/memory/gctransform/test/test_framework.py b/rpython/memory/gctransform/test/test_framework.py
--- a/rpython/memory/gctransform/test/test_framework.py
+++ b/rpython/memory/gctransform/test/test_framework.py
@@ -174,6 +174,7 @@
         return _siphash24('abc')
 
     t = rtype(entrypoint, [s_list_of_strings])
+    backend_optimizations(t, clever_malloc_removal=False, storesink=True)
     t.config.translation.gc = "minimark"
     cbuild = CStandaloneBuilder(t, entrypoint, t.config,
                                 gcpolicy=FrameworkGcPolicy2)

From pypy.commits at gmail.com  Wed Oct  9 07:22:00 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 09 Oct 2019 04:22:00 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: move fast path from interp_codecs to
 unicodehelper, since several places call that directly
Message-ID: <5d9dc2d8.1c69fb81.23e2e.6d9b@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r97743:40f56f2802c9
Date: 2019-10-09 12:21 +0200
http://bitbucket.org/pypy/pypy/changeset/40f56f2802c9/

Log:	move fast path from interp_codecs to unicodehelper, since several
	places call that directly

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -375,6 +375,14 @@
         return res_utf8, len(res), size
 
 def str_decode_utf8(s, errors, final, errorhandler, allow_surrogates=False):
+    try:
+        # fast version first
+        return s, rutf8.check_utf8(s, allow_surrogates=allow_surrogates), len(s)
+    except rutf8.CheckError:
+        return _str_decode_utf8_slowpath(
+            s, errors, final, errorhandler, allow_surrogates=allow_surrogates)
+
+def _str_decode_utf8_slowpath(s, errors, final, errorhandler, allow_surrogates):
     """ Same as checking for the valid utf8, but we know the utf8 is not
     valid so we're trying to either raise or pack stuff with error handler.
     The key difference is that this is call_may_force
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -737,17 +737,10 @@
         errors = 'strict'
     final = space.is_true(w_final)
     state = space.fromcache(CodecState)
-    # call the fast version for checking
-    try:
-        lgt = rutf8.check_utf8(string, allow_surrogates=False)
-    except rutf8.CheckError:
-        res, lgt, pos = unicodehelper.str_decode_utf8(string,
-            errors, final, state.decode_error_handler)
-        return space.newtuple([space.newutf8(res, lgt),
-                               space.newint(pos)])
-    else:
-        return space.newtuple([space.newutf8(string, lgt),
-                               space.newint(len(string))])
+    res, lgt, pos = unicodehelper.str_decode_utf8(string,
+        errors, final, state.decode_error_handler)
+    return space.newtuple([space.newutf8(res, lgt),
+                           space.newint(pos)])
 
 @unwrap_spec(data='bufferstr', errors='text_or_none', byteorder=int,
              w_final=WrappedDefault(False))

From pypy.commits at gmail.com  Wed Oct  9 09:48:21 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 09 Oct 2019 06:48:21 -0700 (PDT)
Subject: [pypy-commit] pypy default: support utf8_len_w in the fake object
 space
Message-ID: <5d9de525.1c69fb81.b1c9b.7190@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r97744:a511d86377d6
Date: 2019-10-09 15:35 +0200
http://bitbucket.org/pypy/pypy/changeset/a511d86377d6/

Log:	support utf8_len_w in the fake object space

diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -149,6 +149,9 @@
         is_root(w_obj)
         return NonConstant(False)
 
+    def utf8_len_w(self, space):
+        return NonConstant((NonConstant("utf8len_foobar"), NonConstant(14)))
+
     @not_rpython
     def unwrap(self, w_obj):
         raise NotImplementedError
diff --git a/pypy/objspace/fake/test/test_objspace.py b/pypy/objspace/fake/test/test_objspace.py
--- a/pypy/objspace/fake/test/test_objspace.py
+++ b/pypy/objspace/fake/test/test_objspace.py
@@ -1,4 +1,5 @@
-import py
+import pytest
+from rpython.rlib.nonconst import NonConstant
 from pypy.objspace.fake.objspace import FakeObjSpace, W_Root
 from pypy.interpreter.argument import Arguments
 from pypy.interpreter.typedef import TypeDef
@@ -63,8 +64,8 @@
     def test_is_true(self):
         space = self.space
         space.translates(lambda: space.is_true(W_Root()))
-        py.test.raises(AssertionError,
-                       space.translates, lambda: space.is_true(42))
+        with pytest.raises(AssertionError):
+            space.translates(lambda: space.is_true(42))
 
     def test_unpackiterable(self):
         space = self.space
@@ -79,3 +80,23 @@
         space = self.space
         space.translates(lambda: (space.get(W_Root(), W_Root()),
                                   space.get(W_Root(), W_Root(), W_Root())))
+
+    def test_bug_utf8_len_w(self):
+        space = self.space
+
+        class A(object):
+            pass
+
+        def f():
+            s = NonConstant('a')
+            w_s = space.newutf8(s, 1)
+            t, l = space.utf8_len_w(w_s)
+            a = A()
+            if l == 1:
+                a.x = 1
+            else:
+                raise Exception
+            return a.x
+        space.translates(f)
+
+

From pypy.commits at gmail.com  Wed Oct  9 09:59:40 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 09 Oct 2019 06:59:40 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: speed up utf8-handling of csv module
Message-ID: <5d9de7cc.1c69fb81.54617.c374@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r97745:1b9016af40bd
Date: 2019-10-09 13:39 +0200
http://bitbucket.org/pypy/pypy/changeset/1b9016af40bd/

Log:	speed up utf8-handling of csv module

diff --git a/pypy/module/_csv/interp_reader.py b/pypy/module/_csv/interp_reader.py
--- a/pypy/module/_csv/interp_reader.py
+++ b/pypy/module/_csv/interp_reader.py
@@ -1,5 +1,5 @@
 from rpython.rlib.rstring import UnicodeBuilder
-from rpython.rlib.rutf8 import Utf8StringIterator
+from rpython.rlib.rutf8 import Utf8StringIterator, Utf8StringBuilder
 from rpython.rlib import objectmodel
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError
@@ -38,16 +38,15 @@
         assert field_builder is not None
         if field_builder.getlength() >= field_limit.limit:
             raise self.error(u"field larger than field limit")
-        field_builder.append(c)
+        field_builder.append_code(c)
 
     def save_field(self, field_builder):
         space = self.space
         field = field_builder.build()
+        w_obj = space.newutf8(field, field_builder.getlength())
         if self.numeric_field:
             self.numeric_field = False
-            w_obj = space.call_function(space.w_float, space.newtext(field))
-        else:
-            w_obj = space.newtext(field)
+            w_obj = space.call_function(space.w_float, w_obj)
         self.fields_w.append(w_obj)
 
     def next_w(self):
@@ -79,13 +78,11 @@
                                  u"(did you open the file in text mode?")
             line = space.utf8_w(w_line)
             for c in Utf8StringIterator(line):
-                # XXX rewrite this to use c (as int) not unichr(c)
-                c = unichr(c)
-                if c == '\0':
+                if c == 0:
                     raise self.error(u"line contains NULL byte")
 
                 if state == START_RECORD:
-                    if c == b'\n' or c == b'\r':
+                    if c == ord(u'\n') or c == ord(u'\r'):
                         state = EAT_CRNL
                         continue
                     # normal character - handle as START_FIELD
@@ -93,23 +90,23 @@
                     # fall-through to the next case
 
                 if state == START_FIELD:
-                    field_builder = UnicodeBuilder(64)
+                    field_builder = Utf8StringBuilder(64)
                     # expecting field
-                    if c == u'\n' or c == u'\r':
+                    if c == ord(u'\n') or c == ord(u'\r'):
                         # save empty field
                         self.save_field(field_builder)
                         state = EAT_CRNL
-                    elif (c == dialect.quotechar and
+                    elif (c == ord(dialect.quotechar) and
                               dialect.quoting != QUOTE_NONE):
                         # start quoted field
                         state = IN_QUOTED_FIELD
-                    elif c == dialect.escapechar:
+                    elif c == ord(dialect.escapechar):
                         # possible escaped character
                         state = ESCAPED_CHAR
-                    elif c == u' ' and dialect.skipinitialspace:
+                    elif c == ord(u' ') and dialect.skipinitialspace:
                         # ignore space at start of field
                         pass
-                    elif c == dialect.delimiter:
+                    elif c == ord(dialect.delimiter):
                         # save empty field
                         self.save_field(field_builder)
                     else:
@@ -120,7 +117,7 @@
                         state = IN_FIELD
 
                 elif state == ESCAPED_CHAR:
-                    if c in '\n\r':
+                    if c == ord(u'\n') or c == ord(u'\r'):
                         self.add_char(field_builder, c)
                         state = AFTER_ESCAPED_CRNL
                     else:
@@ -129,14 +126,14 @@
 
                 elif state == IN_FIELD or state == AFTER_ESCAPED_CRNL:
                     # in unquoted field
-                    if c == u'\n' or c == u'\r':
+                    if c == ord(u'\n') or c == ord(u'\r'):
                         # end of line
                         self.save_field(field_builder)
                         state = EAT_CRNL
-                    elif c == dialect.escapechar:
+                    elif c == ord(dialect.escapechar):
                         # possible escaped character
                         state = ESCAPED_CHAR
-                    elif c == dialect.delimiter:
+                    elif c == ord(dialect.delimiter):
                         # save field - wait for new field
                         self.save_field(field_builder)
                         state = START_FIELD
@@ -146,10 +143,10 @@
 
                 elif state == IN_QUOTED_FIELD:
                     # in quoted field
-                    if c == dialect.escapechar:
+                    if c == ord(dialect.escapechar):
                         # Possible escape character
                         state = ESCAPE_IN_QUOTED_FIELD
-                    elif (c == dialect.quotechar and
+                    elif (c == ord(dialect.quotechar) and
                               dialect.quoting != QUOTE_NONE):
                         if dialect.doublequote:
                             # doublequote; " represented by ""
@@ -168,15 +165,15 @@
                 elif state == QUOTE_IN_QUOTED_FIELD:
                     # doublequote - seen a quote in an quoted field
                     if (dialect.quoting != QUOTE_NONE and
-                            c == dialect.quotechar):
+                            c == ord(dialect.quotechar)):
                         # save "" as "
                         self.add_char(field_builder, c)
                         state = IN_QUOTED_FIELD
-                    elif c == dialect.delimiter:
+                    elif c == ord(dialect.delimiter):
                         # save field - wait for new field
                         self.save_field(field_builder)
                         state = START_FIELD
-                    elif c == u'\n' or c == u'\r':
+                    elif c == ord(u'\n') or c == ord(u'\r'):
                         # end of line
                         self.save_field(field_builder)
                         state = EAT_CRNL
@@ -189,7 +186,7 @@
                             dialect.delimiter, dialect.quotechar))
 
                 elif state == EAT_CRNL:
-                    if not (c == u'\n' or c == u'\r'):
+                    if not (c == ord(u'\n') or c == ord(u'\r')):
                         raise self.error(u"new-line character seen in unquoted "
                                          u"field - do you need to open the file "
                                          u"in universal-newline mode?")
@@ -198,16 +195,16 @@
                 self.save_field(field_builder)
                 break
             elif state == ESCAPED_CHAR:
-                self.add_char(field_builder, u'\n')
+                self.add_char(field_builder, ord(u'\n'))
                 state = IN_FIELD
             elif state == IN_QUOTED_FIELD:
                 pass
             elif state == ESCAPE_IN_QUOTED_FIELD:
-                self.add_char(field_builder, u'\n')
+                self.add_char(field_builder, ord(u'\n'))
                 state = IN_QUOTED_FIELD
             elif state == START_FIELD:
                 # save empty field
-                field_builder = UnicodeBuilder(1)
+                field_builder = Utf8StringBuilder()
                 self.save_field(field_builder)
                 break
             elif state == AFTER_ESCAPED_CRNL:

From pypy.commits at gmail.com  Wed Oct  9 09:59:42 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 09 Oct 2019 06:59:42 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: merge default
Message-ID: <5d9de7ce.1c69fb81.e0333.004c@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r97746:60306f724eeb
Date: 2019-10-09 15:49 +0200
http://bitbucket.org/pypy/pypy/changeset/60306f724eeb/

Log:	merge default

diff --git a/pypy/doc/commandline_ref.rst b/pypy/doc/commandline_ref.rst
--- a/pypy/doc/commandline_ref.rst
+++ b/pypy/doc/commandline_ref.rst
@@ -9,3 +9,4 @@
 
    man/pypy.1.rst
    man/pypy3.1.rst
+   jit_help.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -42,7 +42,8 @@
 -------------------------------
 
 .. toctree::
-   whatsnew-pypy3-head.rst
+    whatsnew-pypy3-head.rst
+    whatsnew-pypy3-7.1.0.rst
 
 CPython 3.5 compatible versions
 -------------------------------
@@ -50,6 +51,8 @@
 .. toctree::
 
    whatsnew-pypy3-7.0.0.rst
+   whatsnew-pypy3-6.0.0.rst
+   whatsnew-pypy3-5.10.0.rst
    whatsnew-pypy3-5.9.0.rst
    whatsnew-pypy3-5.8.0.rst
    whatsnew-pypy3-5.7.0.rst
@@ -62,10 +65,4 @@
    whatsnew-pypy3-5.5.0.rst
    whatsnew-pypy3-5.1.1-alpha1.rst
 
-CPython 3.2 compatible versions
--------------------------------
 
-.. toctree::
-
-   whatsnew-pypy3-2.4.0.rst
-   whatsnew-pypy3-2.3.1.rst
diff --git a/pypy/doc/jit_help.rst b/pypy/doc/jit_help.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/jit_help.rst
@@ -0,0 +1,78 @@
+========
+JIT help
+========
+
+.. note this is from ``pypy --jit help``
+
+Advanced JIT options
+====================
+
+``<pypy> --jit`` [*options*] where *options* is a comma-separated list of
+``OPTION=VALUE``:
+
+ decay=N
+    amount to regularly decay counters by (0=none, 1000=max) (default 40)
+
+ disable_unrolling=N
+    after how many operations we should not unroll (default 200)
+
+ enable_opts=N
+    INTERNAL USE ONLY (MAY NOT WORK OR LEAD TO CRASHES): optimizations to
+    enable, or all =
+    intbounds:rewrite:virtualize:string:pure:earlyforce:heap:unroll (default
+    all)
+
+ function_threshold=N
+    number of times a function must run for it to become traced from start
+    (default 1619)
+
+ inlining=N
+    inline python functions or not (1/0) (default 1)
+
+ loop_longevity=N
+    a parameter controlling how long loops will be kept before being freed,
+    an estimate (default 1000)
+
+ max_retrace_guards=N
+    number of extra guards a retrace can cause (default 15)
+
+ max_unroll_loops=N
+    number of extra unrollings a loop can cause (default 0)
+
+ max_unroll_recursion=N
+    how many levels deep to unroll a recursive function (default 7)
+
+ retrace_limit=N
+    how many times we can try retracing before giving up (default 0)
+
+ threshold=N
+    number of times a loop has to run for it to become hot (default 1039)
+
+ trace_eagerness=N
+    number of times a guard has to fail before we start compiling a bridge
+    (default 200)
+
+ trace_limit=N
+    number of recorded operations before we abort tracing with ABORT_TOO_LONG
+    (default 6000)
+
+ vec=N
+    turn on the vectorization optimization (vecopt). Supports x86 (SSE 4.1),
+    powerpc (SVX), s390x SIMD (default 0)
+
+ vec_all=N
+    try to vectorize trace loops that occur outside of the numpypy library
+    (default 0)
+
+ vec_cost=N
+    threshold for which traces to bail. Unpacking increases the counter,
+    vector operation decrease the cost (default 0)
+
+ off
+    turn off the JIT
+ help
+    print this page
+
+The :ref:`pypyjit<jit-hooks>` module can be used to control the JIT from inside
+pypy
+
diff --git a/pypy/doc/release-v7.2.0.rst b/pypy/doc/release-v7.2.0.rst
--- a/pypy/doc/release-v7.2.0.rst
+++ b/pypy/doc/release-v7.2.0.rst
@@ -307,6 +307,7 @@
 .. _33786 : https://bugs.python.org/issue33786
 .. _32270 : https://bugs.python.org/issue32270
 .. _28691 : https://bugs.python.org/issue28691
+.. _33729 : https://bugs.python.org/issue33729
 
 .. _opencv2: https://github.com/skvark/opencv-python/
 .. _`issue 2617`: https://bitbucket.com/pypy/pypy/issues/2617
diff --git a/pypy/doc/whatsnew-pypy3-2.3.1.rst b/pypy/doc/whatsnew-pypy3-2.3.1.rst
deleted file mode 100644
--- a/pypy/doc/whatsnew-pypy3-2.3.1.rst
+++ /dev/null
@@ -1,6 +0,0 @@
-=========================
-What's new in PyPy3 2.3.1
-=========================
-
-.. this is a revision shortly after pypy3-release-2.3.x
-.. startrev: 0137d8e6657d
diff --git a/pypy/doc/whatsnew-pypy3-2.4.0.rst b/pypy/doc/whatsnew-pypy3-2.4.0.rst
deleted file mode 100644
--- a/pypy/doc/whatsnew-pypy3-2.4.0.rst
+++ /dev/null
@@ -1,6 +0,0 @@
-=========================
-What's new in PyPy3 2.4.0
-=========================
-
-.. this is a revision shortly after pypy3-release-2.4.x
-.. startrev: 12b940544622
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -164,6 +164,9 @@
         is_root(w_obj)
         return NonConstant(False)
 
+    def utf8_len_w(self, space):
+        return NonConstant((NonConstant("utf8len_foobar"), NonConstant(14)))
+
     @not_rpython
     def unwrap(self, w_obj):
         raise NotImplementedError
diff --git a/pypy/objspace/fake/test/test_objspace.py b/pypy/objspace/fake/test/test_objspace.py
--- a/pypy/objspace/fake/test/test_objspace.py
+++ b/pypy/objspace/fake/test/test_objspace.py
@@ -1,4 +1,5 @@
-import py
+import pytest
+from rpython.rlib.nonconst import NonConstant
 from pypy.objspace.fake.objspace import FakeObjSpace, W_Root
 from pypy.interpreter.argument import Arguments
 from pypy.interpreter.typedef import TypeDef
@@ -63,8 +64,8 @@
     def test_is_true(self):
         space = self.space
         space.translates(lambda: space.is_true(W_Root()))
-        py.test.raises(AssertionError,
-                       space.translates, lambda: space.is_true(42))
+        with pytest.raises(AssertionError):
+            space.translates(lambda: space.is_true(42))
 
     def test_unpackiterable(self):
         space = self.space
@@ -79,3 +80,23 @@
         space = self.space
         space.translates(lambda: (space.get(W_Root(), W_Root()),
                                   space.get(W_Root(), W_Root(), W_Root())))
+
+    def test_bug_utf8_len_w(self):
+        space = self.space
+
+        class A(object):
+            pass
+
+        def f():
+            s = NonConstant('a')
+            w_s = space.newutf8(s, 1)
+            t, l = space.utf8_len_w(w_s)
+            a = A()
+            if l == 1:
+                a.x = 1
+            else:
+                raise Exception
+            return a.x
+        space.translates(f)
+
+

From pypy.commits at gmail.com  Wed Oct  9 09:59:44 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 09 Oct 2019 06:59:44 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: remove this hack,
 now that the fake objspace supports utf8_len_w
Message-ID: <5d9de7d0.1c69fb81.1f155.bad2@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r97747:ac3b88caa03e
Date: 2019-10-09 15:57 +0200
http://bitbucket.org/pypy/pypy/changeset/ac3b88caa03e/

Log:	remove this hack, now that the fake objspace supports utf8_len_w

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1787,8 +1787,7 @@
     def convert_arg_to_w_unicode(self, w_obj, strict=None):
         # XXX why convert_to_w_unicode does something slightly different?
         from pypy.objspace.std.unicodeobject import W_UnicodeObject
-        # for z_translation tests
-        if hasattr(self, 'is_fake_objspace'): return self.newtext("foobar")
+        assert not hasattr(self, 'is_fake_objspace')
         return W_UnicodeObject.convert_arg_to_w_unicode(self, w_obj, strict)
 
     def utf8_len_w(self, w_obj):

From pypy.commits at gmail.com  Wed Oct  9 09:59:45 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 09 Oct 2019 06:59:45 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: small improvement: use newlist with a
 sizehint that is the length of the previous line
Message-ID: <5d9de7d1.1c69fb81.1d990.83c7@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r97748:8b92f9a32d07
Date: 2019-10-09 15:58 +0200
http://bitbucket.org/pypy/pypy/changeset/8b92f9a32d07/

Log:	small improvement: use newlist with a sizehint that is the length of
	the previous line

diff --git a/pypy/module/_csv/interp_reader.py b/pypy/module/_csv/interp_reader.py
--- a/pypy/module/_csv/interp_reader.py
+++ b/pypy/module/_csv/interp_reader.py
@@ -1,4 +1,3 @@
-from rpython.rlib.rstring import UnicodeBuilder
 from rpython.rlib.rutf8 import Utf8StringIterator, Utf8StringBuilder
 from rpython.rlib import objectmodel
 from pypy.interpreter.baseobjspace import W_Root
@@ -22,6 +21,7 @@
         self.dialect = dialect
         self.w_iter = w_iter
         self.line_num = 0
+        self.sizehint = 1  # just used for first line
 
     def iter_w(self):
         return self
@@ -52,7 +52,7 @@
     def next_w(self):
         space = self.space
         dialect = self.dialect
-        self.fields_w = []
+        self.fields_w = objectmodel.newlist_hint(self.sizehint)
         self.numeric_field = False
         field_builder = None  # valid iff state not in [START_RECORD, EAT_CRNL]
         state = START_RECORD
@@ -213,6 +213,8 @@
                 break
         #
         w_result = space.newlist(self.fields_w)
+        # assume all lines have the same number of fields
+        self.sizehint = len(self.fields_w)
         self.fields_w = None
         return w_result
 

From pypy.commits at gmail.com  Wed Oct  9 09:59:47 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 09 Oct 2019 06:59:47 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: switch to Utf8StringBuilder for the csv
 writer as well
Message-ID: <5d9de7d3.1c69fb81.cbfc5.ca7d@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r97749:6d2f8470165b
Date: 2019-10-09 15:58 +0200
http://bitbucket.org/pypy/pypy/changeset/6d2f8470165b/

Log:	switch to Utf8StringBuilder for the csv writer as well

diff --git a/pypy/module/_csv/interp_csv.py b/pypy/module/_csv/interp_csv.py
--- a/pypy/module/_csv/interp_csv.py
+++ b/pypy/module/_csv/interp_csv.py
@@ -1,3 +1,4 @@
+from rpython.rlib import rutf8
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.typedef import TypeDef, interp_attrproperty
@@ -47,24 +48,26 @@
     if w_src is None:
         return default
     try:
-        return space.realunicode_w(w_src)
+        return space.text_w(w_src)
     except OperationError as e:
         if e.match(space, space.w_TypeError):
             raise oefmt(space.w_TypeError, '"%s" must be a string', attrname)
         raise
 
-def _get_char(space, w_src, default, name):
+def _get_codepoint(space, w_src, default, name):
     if w_src is None:
         return default
     if space.is_w(w_src, space.w_None):
-        return u'\0'
+        return 0
     if not space.isinstance_w(w_src, space.w_unicode):
         raise oefmt(space.w_TypeError, '"%s" must be string, not %T', name, w_src)
-    src = space.realunicode_w(w_src)
-    if len(src) == 1:
-        return src[0]
+    src, length = space.utf8_len_w(w_src)
+    if length == 1:
+        res = rutf8.codepoint_at_pos(src, 0)
+        assert res >= 0
+        return res
     if len(src) == 0:
-        return u'\0'
+        return 0
     raise oefmt(space.w_TypeError, '"%s" must be a 1-character string', name)
 
 def _build_dialect(space, w_dialect, w_delimiter, w_doublequote,
@@ -104,11 +107,11 @@
             w_strict = _fetch(space, w_dialect, 'strict')
 
     dialect = W_Dialect()
-    dialect.delimiter = _get_char(space, w_delimiter, u',', 'delimiter')
+    dialect.delimiter = _get_codepoint(space, w_delimiter, ord(u','), 'delimiter')
     dialect.doublequote = _get_bool(space, w_doublequote, True)
-    dialect.escapechar = _get_char(space, w_escapechar, u'\0', 'escapechar')
-    dialect.lineterminator = _get_str(space, w_lineterminator, u'\r\n', 'lineterminator')
-    dialect.quotechar = _get_char(space, w_quotechar, u'"', 'quotechar')
+    dialect.escapechar = _get_codepoint(space, w_escapechar, ord(u'\0'), 'escapechar')
+    dialect.lineterminator = _get_str(space, w_lineterminator, '\r\n', 'lineterminator')
+    dialect.quotechar = _get_codepoint(space, w_quotechar, ord(u'"'), 'quotechar')
     tmp_quoting = _get_int(space, w_quoting, QUOTE_MINIMAL, 'quoting')
     dialect.skipinitialspace = _get_bool(space, w_skipinitialspace, False)
     dialect.strict = _get_bool(space, w_strict, False)
@@ -117,13 +120,13 @@
     if not (0 <= tmp_quoting < 4):
         raise oefmt(space.w_TypeError, 'bad "quoting" value')
 
-    if dialect.delimiter == u'\0':
+    if dialect.delimiter == 0:
         raise oefmt(space.w_TypeError,
                     '"delimiter" must be a 1-character string')
 
     if space.is_w(w_quotechar, space.w_None) and w_quoting is None:
         tmp_quoting = QUOTE_NONE
-    if tmp_quoting != QUOTE_NONE and dialect.quotechar == u'\0':
+    if tmp_quoting != QUOTE_NONE and dialect.quotechar == 0:
         raise oefmt(space.w_TypeError,
                     "quotechar must be set if quoting enabled")
     dialect.quoting = tmp_quoting
@@ -158,14 +161,20 @@
 
 
 def _get_escapechar(space, dialect):
-    if dialect.escapechar == u'\0':
+    if dialect.escapechar == 0:
         return space.w_None
-    return space.newtext(dialect.escapechar)
+    s = rutf8.unichr_as_utf8(dialect.escapechar)
+    return space.newutf8(s, 1)
 
 def _get_quotechar(space, dialect):
-    if dialect.quotechar == u'\0':
+    if dialect.quotechar == 0:
         return space.w_None
-    return space.newtext(dialect.quotechar)
+    s = rutf8.unichr_as_utf8(dialect.quotechar)
+    return space.newutf8(s, 1)
+
+def _get_delimiter(space, dialect):
+    s = rutf8.unichr_as_utf8(dialect.delimiter)
+    return space.newutf8(s, 1)
 
 
 W_Dialect.typedef = TypeDef(
@@ -173,8 +182,7 @@
         __new__ = interp2app(W_Dialect___new__),
         __reduce_ex__ = interp2app(W_Dialect.reduce_ex_w),
 
-        delimiter        = interp_attrproperty('delimiter', W_Dialect,
-            wrapfn='newtext'),
+        delimiter        = GetSetProperty(_get_delimiter, cls=W_Dialect),
         doublequote      = interp_attrproperty('doublequote', W_Dialect,
             wrapfn='newbool'),
         escapechar       = GetSetProperty(_get_escapechar, cls=W_Dialect),
diff --git a/pypy/module/_csv/interp_reader.py b/pypy/module/_csv/interp_reader.py
--- a/pypy/module/_csv/interp_reader.py
+++ b/pypy/module/_csv/interp_reader.py
@@ -96,17 +96,17 @@
                         # save empty field
                         self.save_field(field_builder)
                         state = EAT_CRNL
-                    elif (c == ord(dialect.quotechar) and
+                    elif (c == dialect.quotechar and
                               dialect.quoting != QUOTE_NONE):
                         # start quoted field
                         state = IN_QUOTED_FIELD
-                    elif c == ord(dialect.escapechar):
+                    elif c == dialect.escapechar:
                         # possible escaped character
                         state = ESCAPED_CHAR
                     elif c == ord(u' ') and dialect.skipinitialspace:
                         # ignore space at start of field
                         pass
-                    elif c == ord(dialect.delimiter):
+                    elif c == dialect.delimiter:
                         # save empty field
                         self.save_field(field_builder)
                     else:
@@ -130,10 +130,10 @@
                         # end of line
                         self.save_field(field_builder)
                         state = EAT_CRNL
-                    elif c == ord(dialect.escapechar):
+                    elif c == dialect.escapechar:
                         # possible escaped character
                         state = ESCAPED_CHAR
-                    elif c == ord(dialect.delimiter):
+                    elif c == dialect.delimiter:
                         # save field - wait for new field
                         self.save_field(field_builder)
                         state = START_FIELD
@@ -143,10 +143,10 @@
 
                 elif state == IN_QUOTED_FIELD:
                     # in quoted field
-                    if c == ord(dialect.escapechar):
+                    if c == dialect.escapechar:
                         # Possible escape character
                         state = ESCAPE_IN_QUOTED_FIELD
-                    elif (c == ord(dialect.quotechar) and
+                    elif (c == dialect.quotechar and
                               dialect.quoting != QUOTE_NONE):
                         if dialect.doublequote:
                             # doublequote; " represented by ""
@@ -165,11 +165,11 @@
                 elif state == QUOTE_IN_QUOTED_FIELD:
                     # doublequote - seen a quote in an quoted field
                     if (dialect.quoting != QUOTE_NONE and
-                            c == ord(dialect.quotechar)):
+                            c == dialect.quotechar):
                         # save "" as "
                         self.add_char(field_builder, c)
                         state = IN_QUOTED_FIELD
-                    elif c == ord(dialect.delimiter):
+                    elif c == dialect.delimiter:
                         # save field - wait for new field
                         self.save_field(field_builder)
                         state = START_FIELD
@@ -183,7 +183,7 @@
                     else:
                         # illegal
                         raise self.error(u"'%s' expected after '%s'" % (
-                            dialect.delimiter, dialect.quotechar))
+                            unichr(dialect.delimiter), unichr(dialect.quotechar)))
 
                 elif state == EAT_CRNL:
                     if not (c == ord(u'\n') or c == ord(u'\r')):
diff --git a/pypy/module/_csv/interp_writer.py b/pypy/module/_csv/interp_writer.py
--- a/pypy/module/_csv/interp_writer.py
+++ b/pypy/module/_csv/interp_writer.py
@@ -1,4 +1,4 @@
-from rpython.rlib.rstring import UnicodeBuilder
+from rpython.rlib.rutf8 import Utf8StringIterator, Utf8StringBuilder
 from rpython.rlib import objectmodel
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError
@@ -15,11 +15,13 @@
         self.dialect = dialect
         self.w_filewrite = space.getattr(w_fileobj, space.newtext('write'))
         # precompute this
-        special = dialect.delimiter + dialect.lineterminator
-        if dialect.escapechar != '\0':
-            special += dialect.escapechar
-        if dialect.quotechar != '\0':
-            special += dialect.quotechar
+        special = [dialect.delimiter]
+        for c in Utf8StringIterator(dialect.lineterminator):
+            special.append(c)
+        if dialect.escapechar != 0:
+            special.append(dialect.escapechar)
+        if dialect.quotechar != 0:
+            special.append(dialect.quotechar)
         self.special_characters = special
 
     @objectmodel.dont_inline
@@ -35,16 +37,17 @@
         space = self.space
         fields_w = space.listview(w_fields)
         dialect = self.dialect
-        rec = UnicodeBuilder(80)
+        rec = Utf8StringBuilder(80)
         #
         for field_index in range(len(fields_w)):
             w_field = fields_w[field_index]
             if space.is_w(w_field, space.w_None):
-                field = u""
+                field = ""
+                length = 0
             elif space.isinstance_w(w_field, space.w_float):
-                field = space.realunicode_w(space.repr(w_field))
+                field, length = space.utf8_len_w(space.repr(w_field))
             else:
-                field = space.realunicode_w(space.str(w_field))
+                field, length = space.utf8_len_w(space.str(w_field))
             #
             if dialect.quoting == QUOTE_NONNUMERIC:
                 try:
@@ -57,9 +60,9 @@
             elif dialect.quoting == QUOTE_ALL:
                 quoted = True
             elif dialect.quoting == QUOTE_MINIMAL:
-                # Find out if we really quoting
+                # Find out if we really need quoting.
                 special_characters = self.special_characters
-                for c in field:
+                for c in Utf8StringIterator(field):
                     if c in special_characters:
                         if c != dialect.quotechar or dialect.doublequote:
                             quoted = True
@@ -78,15 +81,15 @@
 
             # If this is not the first field we need a field separator
             if field_index > 0:
-                rec.append(dialect.delimiter)
+                rec.append_code(dialect.delimiter)
 
             # Handle preceding quote
             if quoted:
-                rec.append(dialect.quotechar)
+                rec.append_code(dialect.quotechar)
 
             # Copy field data
             special_characters = self.special_characters
-            for c in field:
+            for c in Utf8StringIterator(field):
                 if c in special_characters:
                     if dialect.quoting == QUOTE_NONE:
                         want_escape = True
@@ -94,28 +97,28 @@
                         want_escape = False
                         if c == dialect.quotechar:
                             if dialect.doublequote:
-                                rec.append(dialect.quotechar)
+                                rec.append_code(dialect.quotechar)
                             else:
                                 want_escape = True
                     if want_escape:
-                        if dialect.escapechar == u'\0':
+                        if dialect.escapechar == 0:
                             raise self.error("need to escape, "
                                              "but no escapechar set")
-                        rec.append(dialect.escapechar)
+                        rec.append_code(dialect.escapechar)
                     else:
                         assert quoted
                 # Copy field character into record buffer
-                rec.append(c)
+                rec.append_code(c)
 
             # Handle final quote
             if quoted:
-                rec.append(dialect.quotechar)
+                rec.append_code(dialect.quotechar)
 
         # Add line terminator
         rec.append(dialect.lineterminator)
 
         line = rec.build()
-        return space.call_function(self.w_filewrite, space.newtext(line))
+        return space.call_function(self.w_filewrite, space.newutf8(line, rec.getlength()))
 
     def writerows(self, w_seqseq):
         """Construct and write a series of sequences to a csv file.

From pypy.commits at gmail.com  Wed Oct  9 11:01:48 2019
From: pypy.commits at gmail.com (arigo)
Date: Wed, 09 Oct 2019 08:01:48 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: Update link to the MS page for
 vcruntime140.dll
Message-ID: <5d9df65c.1c69fb81.4d22c.2657@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r953:b97894caf276
Date: 2019-10-09 16:58 +0200
http://bitbucket.org/pypy/pypy.org/changeset/b97894caf276/

Log:	Update link to the MS page for vcruntime140.dll

diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -115,8 +115,8 @@
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.1-linux64.tar.bz2">Linux x86-64 binary (64bit, built on Ubuntu 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.1-osx64.tar.bz2">Mac OS X binary (64bit)</a></li>
 <li>FreeBSD x86 and x86_64: see <a class="reference external" href="http://www.freshports.org/lang/pypy">FreshPorts</a></li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.1-win32.zip">Windows binary (32bit)</a> (you might need the VS 2008 runtime library
-installer <a class="reference external" href="http://www.microsoft.com/en-us/download/details.aspx?id=5582">vcredist_x86.exe</a>.)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.1-win32.zip">Windows binary (32bit)</a> (you might need the VC runtime library
+installer <a class="reference external" href="https://www.microsoft.com/en-us/download/details.aspx?id=52685">vcredist.x86.exe</a>.)</li>
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.1-ppc64.tar.bz2">PowerPC PPC64 Linux binary (64bit big-endian, Fedora 20)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.1-ppc64le.tar.bz2">PowerPC PPC64le Linux binary (64bit little-endian, Fedora 21)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.1-s390x.tar.bz2">s390x Linux binary (built on Redhat Linux 7.2)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
@@ -131,7 +131,8 @@
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.1-linux64.tar.bz2">Linux x86-64 binary (64bit, built on Ubuntu 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.1-linux32.tar.bz2">Linux x86 binary (32bit, built on Ubuntu 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.1-osx64.tar.bz2">Mac OS X binary (64bit)</a> (High Sierra &gt;= 10.13, not for Sierra and below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.1-win32.zip">Windows binary (32bit)</a></li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.1-win32.zip">Windows binary (32bit)</a> (you might need the VC runtime library
+installer <a class="reference external" href="https://www.microsoft.com/en-us/download/details.aspx?id=52685">vcredist.x86.exe</a>.)</li>
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.1-ppc64.tar.bz2">PowerPC PPC64 Linux binary (64bit big-endian, Fedora 20)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.1-ppc64le.tar.bz2">PowerPC PPC64le Linux binary (64bit little-endian, Fedora 21)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.1-s390x.tar.bz2">s390x Linux binary (built on Redhat Linux 7.2)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
@@ -146,7 +147,8 @@
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.5-v7.0.0-linux32.tar.bz2">Linux x86 binary (32bit, built on Ubuntu 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.5-v7.0.0-linux64.tar.bz2">Linux x86-64 binary (64bit, built on Ubuntu 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.5-v7.0.0-osx64.tar.bz2">Mac OS X binary (64bit)</a> (High Sierra &gt;= 10.13, not for Sierra and below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.5-v7.0.0-win32.zip">Windows binary (32bit)</a> <strong>BETA</strong></li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.5-v7.0.0-win32.zip">Windows binary (32bit)</a> <strong>BETA</strong> (you might need the VC runtime library
+installer <a class="reference external" href="https://www.microsoft.com/en-us/download/details.aspx?id=52685">vcredist.x86.exe</a>.)</li>
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.5-v7.0.0-ppc64.tar.bz2">PowerPC PPC64 Linux binary (64bit big-endian, Fedora 20)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.5-v7.0.0-ppc64le.tar.bz2">PowerPC PPC64le Linux binary (64bit little-endian, Fedora 21)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.5-v7.0.0-s390x.tar.bz2">s390x Linux binary (built on Redhat Linux 7.2)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -88,8 +88,8 @@
 * `Linux x86-64 binary (64bit, built on Ubuntu 14.04)`__ (see ``[1]`` below)
 * `Mac OS X binary (64bit)`__
 * FreeBSD x86 and x86_64: see FreshPorts_
-* `Windows binary (32bit)`__ (you might need the VS 2008 runtime library
-  installer `vcredist_x86.exe`_.)
+* `Windows binary (32bit)`__ (you might need the VC runtime library
+  installer `vcredist.x86.exe`_.)
 * `PowerPC PPC64 Linux binary (64bit big-endian, Fedora 20)`__ (see ``[1]`` below)
 * `PowerPC PPC64le Linux binary (64bit little-endian, Fedora 21)`__ (see ``[1]`` below)
 * `s390x Linux binary (built on Redhat Linux 7.2)`__ (see ``[1]`` below)
@@ -106,7 +106,7 @@
 .. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.1-s390x.tar.bz2
 .. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.1-src.tar.bz2
 .. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.1-src.zip
-.. _`vcredist_x86.exe`: http://www.microsoft.com/en-us/download/details.aspx?id=5582
+.. _`vcredist.x86.exe`: https://www.microsoft.com/en-us/download/details.aspx?id=52685
 .. __: https://bitbucket.org/pypy/pypy/downloads
 .. _mirror: http://buildbot.pypy.org/mirror/
 .. _FreshPorts: http://www.freshports.org/lang/pypy
@@ -120,7 +120,8 @@
 * `Linux x86-64 binary (64bit, built on Ubuntu 14.04)`__ (see ``[1]`` below)
 * `Linux x86 binary (32bit, built on Ubuntu 14.04)`__ (see ``[1]`` below)
 * `Mac OS X binary (64bit)`__ (High Sierra >= 10.13, not for Sierra and below)
-* `Windows binary (32bit)`__ 
+* `Windows binary (32bit)`__ (you might need the VC runtime library
+  installer `vcredist.x86.exe`_.)
 * `PowerPC PPC64 Linux binary (64bit big-endian, Fedora 20)`__ (see ``[1]`` below)
 * `PowerPC PPC64le Linux binary (64bit little-endian, Fedora 21)`__ (see ``[1]`` below)
 * `s390x Linux binary (built on Redhat Linux 7.2)`__ (see ``[1]`` below)
@@ -148,7 +149,8 @@
 * `Linux x86 binary (32bit, built on Ubuntu 14.04)`__ (see ``[1]`` below)
 * `Linux x86-64 binary (64bit, built on Ubuntu 14.04)`__ (see ``[1]`` below)
 * `Mac OS X binary (64bit)`__ (High Sierra >= 10.13, not for Sierra and below)
-* `Windows binary (32bit)`__ **BETA** 
+* `Windows binary (32bit)`__ **BETA** (you might need the VC runtime library
+  installer `vcredist.x86.exe`_.)
 * `PowerPC PPC64 Linux binary (64bit big-endian, Fedora 20)`__ (see ``[1]`` below)
 * `PowerPC PPC64le Linux binary (64bit little-endian, Fedora 21)`__ (see ``[1]`` below)
 * `s390x Linux binary (built on Redhat Linux 7.2)`__ (see ``[1]`` below)

From pypy.commits at gmail.com  Wed Oct  9 15:12:11 2019
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 09 Oct 2019 12:12:11 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy3.6-v7.x: Add
 _PyDict_GetItemWithError (part of the public API in py3)
Message-ID: <5d9e310b.1c69fb81.c4ad1.093a@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: release-pypy3.6-v7.x
Changeset: r97750:c162c307e6dd
Date: 2019-10-04 18:02 +0100
http://bitbucket.org/pypy/pypy/changeset/c162c307e6dd/

Log:	Add _PyDict_GetItemWithError (part of the public API in py3)
	(grafted from d663ce56919c62a2bf5e242ee79b8bc3c640662a)

diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -80,6 +80,13 @@
     # XXX this is wrong with IntMutableCell.  Hope it works...
     return w_dict.getitem(w_key)
 
+ at cpython_api([PyObject, PyObject], PyObject, result_borrowed=True)
+def _PyDict_GetItemWithError(space, w_dict, w_key):
+    # Like PyDict_GetItem(), but doesn't swallow the error
+    if not isinstance(w_dict, W_DictMultiObject):
+        PyErr_BadInternalCall(space)
+    return w_dict.getitem(w_key)
+
 @cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)
 def PyDict_SetItem(space, w_dict, w_key, w_obj):
     if not isinstance(w_dict, W_DictMultiObject):
diff --git a/pypy/module/cpyext/test/test_dictobject.py b/pypy/module/cpyext/test/test_dictobject.py
--- a/pypy/module/cpyext/test/test_dictobject.py
+++ b/pypy/module/cpyext/test/test_dictobject.py
@@ -175,6 +175,26 @@
             ])
         assert module.dict_proxy({'a': 1, 'b': 2}) == 2
 
+    def test_getitemwitherror(self):
+        module = self.import_extension('foo', [
+            ("dict_getitem", "METH_VARARGS",
+             """
+             PyObject *d, *key, *result;
+             if (!PyArg_ParseTuple(args, "OO", &d, &key)) {
+                return NULL;
+             }
+             result = _PyDict_GetItemWithError(d, key);
+             if (result == NULL && !PyErr_Occurred())
+                Py_RETURN_NONE;
+             Py_XINCREF(result);
+             return result;
+             """)])
+        d = {'foo': 'bar'}
+        assert module.dict_getitem(d, 'foo') == 'bar'
+        assert module.dict_getitem(d, 'missing') is None
+        with raises(TypeError):
+            module.dict_getitem(d, [])
+
     def test_setdefault(self):
         module = self.import_extension('foo', [
             ("setdefault", "METH_VARARGS",

From pypy.commits at gmail.com  Wed Oct  9 15:12:13 2019
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 09 Oct 2019 12:12:13 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy3.6-v7.x: Rename to
 PyDict_GetItemWithError (no leading underscore)
Message-ID: <5d9e310d.1c69fb81.ba216.2279@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: release-pypy3.6-v7.x
Changeset: r97751:5da45ced70e5
Date: 2019-10-04 18:29 +0100
http://bitbucket.org/pypy/pypy/changeset/5da45ced70e5/

Log:	Rename to PyDict_GetItemWithError (no leading underscore) (grafted
	from 6e344fc026c3997723683b2ae7ed953fdf84c387)

diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -81,8 +81,11 @@
     return w_dict.getitem(w_key)
 
 @cpython_api([PyObject, PyObject], PyObject, result_borrowed=True)
-def _PyDict_GetItemWithError(space, w_dict, w_key):
-    # Like PyDict_GetItem(), but doesn't swallow the error
+def PyDict_GetItemWithError(space, w_dict, w_key):
+    """Variant of PyDict_GetItem() that does not suppress
+    exceptions. Return NULL with an exception set if an exception
+    occurred.  Return NULL without an exception set if the key
+    wasn't present."""
     if not isinstance(w_dict, W_DictMultiObject):
         PyErr_BadInternalCall(space)
     return w_dict.getitem(w_key)
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -243,14 +243,6 @@
 def PyWrapper_New(space, w_d, w_self):
     raise NotImplementedError
 
- at cpython_api([PyObject, PyObject], PyObject)
-def PyDict_GetItemWithError(space, p, key):
-    """Variant of PyDict_GetItem() that does not suppress
-    exceptions. Return NULL with an exception set if an exception
-    occurred.  Return NULL without an exception set if the key
-    wasn't present."""
-    raise NotImplementedError
-
 @cpython_api([PyObject, PyObject, rffi.INT_real], rffi.INT_real, error=-1)
 def PyDict_MergeFromSeq2(space, a, seq2, override):
     """Update or merge into dictionary a, from the key-value pairs in seq2.
diff --git a/pypy/module/cpyext/test/test_dictobject.py b/pypy/module/cpyext/test/test_dictobject.py
--- a/pypy/module/cpyext/test/test_dictobject.py
+++ b/pypy/module/cpyext/test/test_dictobject.py
@@ -183,7 +183,7 @@
              if (!PyArg_ParseTuple(args, "OO", &d, &key)) {
                 return NULL;
              }
-             result = _PyDict_GetItemWithError(d, key);
+             result = PyDict_GetItemWithError(d, key);
              if (result == NULL && !PyErr_Occurred())
                 Py_RETURN_NONE;
              Py_XINCREF(result);

From pypy.commits at gmail.com  Wed Oct  9 15:12:15 2019
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 09 Oct 2019 12:12:15 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy2.7-v7.x: Add
 _PyDict_GetItemWithError (part of the public API in py3)
Message-ID: <5d9e310f.1c69fb81.ae168.9368@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: release-pypy2.7-v7.x
Changeset: r97752:4a68d8d3d2fc
Date: 2019-10-04 18:02 +0100
http://bitbucket.org/pypy/pypy/changeset/4a68d8d3d2fc/

Log:	Add _PyDict_GetItemWithError (part of the public API in py3)
	(grafted from d663ce56919c62a2bf5e242ee79b8bc3c640662a)

diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -80,6 +80,13 @@
     # XXX this is wrong with IntMutableCell.  Hope it works...
     return w_dict.getitem(w_key)
 
+ at cpython_api([PyObject, PyObject], PyObject, result_borrowed=True)
+def _PyDict_GetItemWithError(space, w_dict, w_key):
+    # Like PyDict_GetItem(), but doesn't swallow the error
+    if not isinstance(w_dict, W_DictMultiObject):
+        PyErr_BadInternalCall(space)
+    return w_dict.getitem(w_key)
+
 @cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)
 def PyDict_SetItem(space, w_dict, w_key, w_obj):
     if not isinstance(w_dict, W_DictMultiObject):
diff --git a/pypy/module/cpyext/test/test_dictobject.py b/pypy/module/cpyext/test/test_dictobject.py
--- a/pypy/module/cpyext/test/test_dictobject.py
+++ b/pypy/module/cpyext/test/test_dictobject.py
@@ -173,6 +173,26 @@
             ])
         assert module.dict_proxy({'a': 1, 'b': 2}) == 2
 
+    def test_getitemwitherror(self):
+        module = self.import_extension('foo', [
+            ("dict_getitem", "METH_VARARGS",
+             """
+             PyObject *d, *key, *result;
+             if (!PyArg_ParseTuple(args, "OO", &d, &key)) {
+                return NULL;
+             }
+             result = _PyDict_GetItemWithError(d, key);
+             if (result == NULL && !PyErr_Occurred())
+                Py_RETURN_NONE;
+             Py_XINCREF(result);
+             return result;
+             """)])
+        d = {'foo': 'bar'}
+        assert module.dict_getitem(d, 'foo') == 'bar'
+        assert module.dict_getitem(d, 'missing') is None
+        with raises(TypeError):
+            module.dict_getitem(d, [])
+
     def test_update(self):
         module = self.import_extension('foo', [
             ("update", "METH_VARARGS",

From pypy.commits at gmail.com  Wed Oct  9 15:12:17 2019
From: pypy.commits at gmail.com (mattip)
Date: Wed, 09 Oct 2019 12:12:17 -0700 (PDT)
Subject: [pypy-commit] pypy default: update release note
Message-ID: <5d9e3111.1c69fb81.ceec.040f@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r97753:46cfab6a4a1e
Date: 2019-10-09 22:11 +0300
http://bitbucket.org/pypy/pypy/changeset/46cfab6a4a1e/

Log:	update release note

diff --git a/pypy/doc/release-v7.2.0.rst b/pypy/doc/release-v7.2.0.rst
--- a/pypy/doc/release-v7.2.0.rst
+++ b/pypy/doc/release-v7.2.0.rst
@@ -216,6 +216,7 @@
 * Add more constants to `sysconfig``. Set ``MACOSX_DEPLOYMENT_TARGET`` for
   darwin (`issue 2994`_)
 * fix ``CBuffer.buffer_attach``
+* Add ``_PyDict_GetItemWithError`` (``PyDict_GetItemWithError`` on Python3)
 
 Python 3.6 only
 ---------------

From pypy.commits at gmail.com  Thu Oct 10 09:44:28 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 10 Oct 2019 06:44:28 -0700 (PDT)
Subject: [pypy-commit] pypy default: debug-print quasi-immutable
 invalidations
Message-ID: <5d9f35bc.1c69fb81.ad419.7932@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r97754:5dd2d3cca75b
Date: 2019-10-10 15:43 +0200
http://bitbucket.org/pypy/pypy/changeset/5dd2d3cca75b/

Log:	debug-print quasi-immutable invalidations

	(been wanting that since a while)

diff --git a/rpython/jit/metainterp/quasiimmut.py b/rpython/jit/metainterp/quasiimmut.py
--- a/rpython/jit/metainterp/quasiimmut.py
+++ b/rpython/jit/metainterp/quasiimmut.py
@@ -5,6 +5,7 @@
 from rpython.jit.metainterp.history import (
     AbstractDescr, ConstPtr, ConstInt, ConstFloat)
 from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop
 
 
 def get_mutate_field_name(fieldname):
@@ -31,7 +32,7 @@
         qmut_ptr = getattr(p, mutatefieldname)
         setattr(p, mutatefieldname, lltype.nullptr(rclass.OBJECT))
         qmut = cast_base_ptr_to_instance(QuasiImmut, qmut_ptr)
-        qmut.invalidate()
+        qmut.invalidate(mutatefieldname)
     _invalidate_now._dont_inline_ = True
     #
     def invalidation(p):
@@ -45,7 +46,7 @@
     if qmut_ref:
         cpu.bh_setfield_gc_r(p, ConstPtr.value, mutatefielddescr)
         qmut = cast_gcref_to_instance(QuasiImmut, qmut_ref)
-        qmut.invalidate()
+        qmut.invalidate(mutatefielddescr.fieldname)
 
 
 class QuasiImmut(object):
@@ -78,7 +79,8 @@
         # already invalidated; see below
         self.compress_limit = (len(self.looptokens_wrefs) + 15) * 2
 
-    def invalidate(self):
+    def invalidate(self, fieldname=None):
+        debug_start("jit-invalidate-quasi-immutable")
         # When this is called, all the loops that we record become
         # invalid: all GUARD_NOT_INVALIDATED in these loops (and
         # in attached bridges) must now fail.
@@ -87,9 +89,11 @@
             return
         wrefs = self.looptokens_wrefs
         self.looptokens_wrefs = []
+        invalidated = 0
         for wref in wrefs:
             looptoken = wref()
             if looptoken is not None:
+                invalidated += 1
                 looptoken.invalidated = True
                 self.cpu.invalidate_loop(looptoken)
                 # NB. we must call cpu.invalidate_loop() even if
@@ -100,6 +104,8 @@
                 if not we_are_translated():
                     self.cpu.stats.invalidated_token_numbers.add(
                         looptoken.number)
+        debug_print("fieldname", fieldname or "<unknown>", "invalidated", invalidated)
+        debug_stop("jit-invalidate-quasi-immutable")
 
 
 class QuasiImmutDescr(AbstractDescr):

From pypy.commits at gmail.com  Thu Oct 10 12:10:32 2019
From: pypy.commits at gmail.com (mattip)
Date: Thu, 10 Oct 2019 09:10:32 -0700 (PDT)
Subject: [pypy-commit] pypy default: Added tags for v7.2.0rc2
Message-ID: <5d9f57f8.1c69fb81.15246.ac70@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r97755:e1741b330c9d
Date: 2019-10-10 18:53 +0300
http://bitbucket.org/pypy/pypy/changeset/e1741b330c9d/

Log:	Added tags for v7.2.0rc2

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -53,3 +53,5 @@
 85dae4fd5c234b482feff834c73e089872194541 release-pypy2.7-v7.2.0rc0
 7ffb92269488f37c707ce66076f50ffd8613f8e2 release-pypy3.6-v7.2.0rc0
 4d6761df14ffd6f38450f183ac1fad32c946c21b release-pypy3.6-v7.2.0rc1
+5da45ced70e515f94686be0df47c59abd1348ebc release-pypy3.6-v7.2.0rc2
+4a68d8d3d2fc1faec2e83bcb4d28559099092574 release-pypy2.7-v7.2.0rc2

From pypy.commits at gmail.com  Thu Oct 10 14:12:46 2019
From: pypy.commits at gmail.com (stevie_92)
Date: Thu, 10 Oct 2019 11:12:46 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-gc-cycle: Check state during cpyext rrc
 gc processing (incorrect state after garbage handling)
Message-ID: <5d9f749e.1c69fb81.f6731.207d@mx.google.com>

Author: Stefan Beyer <home at sbeyer.at>
Branch: cpyext-gc-cycle
Changeset: r97756:30c564b06dd8
Date: 2019-10-10 20:12 +0200
http://bitbucket.org/pypy/pypy/changeset/30c564b06dd8/

Log:	Check state during cpyext rrc gc processing (incorrect state after
	garbage handling) Handle non-gc objects in o_list correctly Fixed
	handling of gc_refs in rrc incmark Fixed issue with pypy_link of
	freed objects in p_list in rrc incmark

diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -321,61 +321,63 @@
                                              finalize, from_ref, cts)
     from pypy.module.cpyext.api import generic_cpy_call
 
-    start = time.time()
 
-    while True:
-        py_obj = rawrefcount.next_dead(PyObject)
-        if not py_obj:
-            break
-        decref(space, py_obj)
+    if rawrefcount.check_state():
+        start = time.time()
 
-    while True:
-        py_obj = rawrefcount.next_cyclic_isolate(PyObject)
-        if not py_obj:
-            break
-        finalize(space, py_obj)
+        while True:
+            py_obj = rawrefcount.next_dead(PyObject)
+            if not py_obj:
+                break
+            decref(space, py_obj)
 
-    while True:
-        py_obj = rawrefcount.cyclic_garbage_head(PyObject)
-        if not py_obj:
-            break
-        pyobj = rffi.cast(PyObject, py_obj)
-        adr_int = llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(pyobj))
-        pto = pyobj.c_ob_type
-        if pto.c_tp_clear:
-            incref(space, py_obj)
-            #if pto and pto.c_tp_name:
-            #    tp_name = pto.c_tp_name
-            #    name = rffi.charp2str(cts.cast('char*', tp_name))
-            #    debug_print("tp_clear", pyobj, ": type", pto,
-            #                ": name", name)
-            generic_cpy_call(space, pto.c_tp_clear, pyobj)
-            decref(space, py_obj)
-        head = rawrefcount.cyclic_garbage_head(PyObject)
-        if adr_int == llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(head)):
-            rawrefcount.cyclic_garbage_remove()
+        while True:
+            py_obj = rawrefcount.next_cyclic_isolate(PyObject)
+            if not py_obj:
+                break
+            finalize(space, py_obj)
 
-    rawrefcount.begin_garbage()
-    w_list = space.newlist([])
-    while True:
-        w_obj = rawrefcount.next_garbage_pypy(W_Root)
-        if not py_obj:
-            break
-        w_list.append(w_obj)
-    while True:
-        w_pyobj = rawrefcount.next_garbage_pyobj(PyObject)
-        if not w_pyobj:
-            break
-        w_obj = from_ref(space, w_pyobj)
-        w_list.append(w_obj)
-    space.setattr(space.builtin_modules['gc'], space.newtext('garbage'),
-                  w_list)
-    rawrefcount.end_garbage()
+        while True:
+            py_obj = rawrefcount.cyclic_garbage_head(PyObject)
+            if not py_obj:
+                break
+            pyobj = rffi.cast(PyObject, py_obj)
+            adr_int = llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(pyobj))
+            pto = pyobj.c_ob_type
+            if pto.c_tp_clear:
+                incref(space, py_obj)
+                #if pto and pto.c_tp_name:
+                #    tp_name = pto.c_tp_name
+                #    name = rffi.charp2str(cts.cast('char*', tp_name))
+                #    debug_print("tp_clear", pyobj, ": type", pto,
+                #                ": name", name)
+                generic_cpy_call(space, pto.c_tp_clear, pyobj)
+                decref(space, py_obj)
+            head = rawrefcount.cyclic_garbage_head(PyObject)
+            if adr_int == llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(head)):
+                rawrefcount.cyclic_garbage_remove()
 
-    duration = time.time() - start
-    module = space.builtin_modules['gc']
-    durations = space.getattr(module, space.newtext('cpyext_durations'))
-    durations.append(space.newfloat(duration))
+        rawrefcount.begin_garbage()
+        w_list = space.newlist([])
+        while True:
+            w_obj = rawrefcount.next_garbage_pypy(W_Root)
+            if not py_obj:
+                break
+            w_list.append(w_obj)
+        while True:
+            w_pyobj = rawrefcount.next_garbage_pyobj(PyObject)
+            if not w_pyobj:
+                break
+            w_obj = from_ref(space, w_pyobj)
+            w_list.append(w_obj)
+        space.setattr(space.builtin_modules['gc'], space.newtext('garbage'),
+                      w_list)
+        rawrefcount.end_garbage()
+
+        duration = time.time() - start
+        module = space.builtin_modules['gc']
+        durations = space.getattr(module, space.newtext('cpyext_durations'))
+        durations.append(space.newfloat(duration))
 
 class PyObjDeallocAction(executioncontext.AsyncAction):
     """An action that invokes _Py_Dealloc() on the dying PyObjects.
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -71,7 +71,7 @@
 from rpython.memory.support import mangle_hash
 from rpython.rlib.rarithmetic import ovfcheck, LONG_BIT, intmask, r_uint
 from rpython.rlib.rarithmetic import LONG_BIT_SHIFT
-from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop
+from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop, debug_flush
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib import rgc
 from rpython.memory.gc.minimarkpage import out_of_memory
@@ -1292,6 +1292,7 @@
 
     def debug_check_consistency(self):
         if self.DEBUG:
+            debug_flush() # TODO: remove?
             ll_assert(not self.young_rawmalloced_objects,
                       "young raw-malloced objects in a major collection")
             ll_assert(not self.young_objects_with_weakrefs.non_empty(),
@@ -2025,10 +2026,11 @@
             #
             # Check that the flags are correct: we must not have
             # GCFLAG_TRACK_YOUNG_PTRS so far.
-            ll_assert(self.header(obj).tid & GCFLAG_TRACK_YOUNG_PTRS == 0,
-                      "old_objects_pointing_to_young contains obj with "
-                      "GCFLAG_TRACK_YOUNG_PTRS")
-            #
+            # TODO: fix
+            #ll_assert(self.header(obj).tid & GCFLAG_TRACK_YOUNG_PTRS == 0,
+            #          "old_objects_pointing_to_young contains obj with "
+            #          "GCFLAG_TRACK_YOUNG_PTRS")
+
             # Add the flag GCFLAG_TRACK_YOUNG_PTRS.  All live objects should
             # have this flag set after a nursery collection.
             self.header(obj).tid |= GCFLAG_TRACK_YOUNG_PTRS
@@ -2407,7 +2409,11 @@
                 #         - (non-inc) mark expects all objects to be marked
                 #         - both do not rescan nonstack-roots
                 if self.rrc_enabled:
+                    debug_print("starting rrc state:", self.rrc_gc.state)
+                    debug_print("starting marking_state:", self.rrc_gc.marking_state)
                     rrc_finished = self.rrc_gc.major_collection_trace_step()
+                    debug_print("ending rrc state:", self.rrc_gc.state)
+                    debug_print("ending marking_state:", self.rrc_gc.marking_state)
                 else:
                     rrc_finished = True
 
@@ -3138,6 +3144,10 @@
         ll_assert(self.rrc_enabled, "rawrefcount.init not called")
         self.rrc_gc.mark_deallocating(gcobj, pyobject)
 
+    def rawrefcount_check_state(self):
+        ll_assert(self.rrc_enabled, "rawrefcount.init not called")
+        return self.rrc_gc.state == RawRefCountBaseGC.STATE_DEFAULT
+
     def rawrefcount_next_dead(self):
         ll_assert(self.rrc_enabled, "rawrefcount.init not called")
         return self.rrc_gc.next_dead()
@@ -3156,10 +3166,12 @@
 
     def rawrefcount_begin_garbage(self):
         ll_assert(self.rrc_enabled, "rawrefcount.init not called")
+        debug_print("rrc state garbage")
         self.rrc_gc.state = RawRefCountBaseGC.STATE_GARBAGE
 
     def rawrefcount_end_garbage(self):
         ll_assert(self.rrc_enabled, "rawrefcount.init not called")
+        debug_print("rrc state default")
         self.rrc_gc.state = RawRefCountBaseGC.STATE_DEFAULT
 
     def rawrefcount_next_garbage_pypy(self):
diff --git a/rpython/memory/gc/rrc/base.py b/rpython/memory/gc/rrc/base.py
--- a/rpython/memory/gc/rrc/base.py
+++ b/rpython/memory/gc/rrc/base.py
@@ -81,6 +81,7 @@
     RAWREFCOUNT_REFS_SHIFT = 1
     RAWREFCOUNT_REFS_MASK_FINALIZED = 1
     RAWREFCOUNT_REFS_UNTRACKED = -2 << RAWREFCOUNT_REFS_SHIFT
+    RAWREFCOUNT_REFS_REACHABLE = -3 << RAWREFCOUNT_REFS_SHIFT
 
     def _pyobj(self, pyobjaddr):
         return llmemory.cast_adr_to_ptr(pyobjaddr, self.PYOBJ_HDR_PTR)
@@ -102,7 +103,7 @@
         self.p_dict_nurs = self.gc.AddressDict()  # nursery keys only
         self.dealloc_trigger_callback = dealloc_trigger_callback
         self.dealloc_pending = self.gc.AddressStack()
-        self.refcnt_dict = self.gc.AddressDict()
+        self.pypy_link_dict = self.gc.AddressDict()
         self.tp_traverse = tp_traverse
         self.pyobj_list = self._pygchdr(pyobj_list)
         self.tuple_list = self._pygchdr(tuple_list)
@@ -118,6 +119,7 @@
         self.finalizer_type = finalizer_type
         self.tuple_maybe_untrack = tuple_maybe_untrack
         self.state = self.STATE_DEFAULT
+        self.marking_state = 0
         self.cycle_enabled = True
         inc_limit = env.read_uint_from_env('PYPY_RRC_GC_INCREMENT_STEP')
         if inc_limit > 0:
@@ -377,7 +379,7 @@
             debug_print("pyobj stays alive", pyobj, "rc", rc, "cyclic_rc",
                         cyclic_rc)
             if use_dict:
-                obj = self.refcnt_dict.get(pyobject)
+                obj = self.pypy_link_dict.get(pyobject)
             else:
                 intobj = pyobj.c_ob_pypy_link
                 obj = llmemory.cast_int_to_adr(intobj)
@@ -404,7 +406,7 @@
             # force the corresponding object to be alive
             debug_print("pyobj stays alive", pyobj, "rc", rc)
             if use_dict:
-                obj = self.refcnt_dict.get(pyobject)
+                obj = self.pypy_link_dict.get(pyobject)
             else:
                 intobj = pyobj.c_ob_pypy_link
                 obj = llmemory.cast_int_to_adr(intobj)
@@ -489,7 +491,7 @@
                 if pyobj.c_ob_pypy_link <> 0:
                     if use_dict:
                         pyobject = llmemory.cast_ptr_to_adr(pyobj)
-                        obj = self.refcnt_dict.get(pyobject)
+                        obj = self.pypy_link_dict.get(pyobject)
                     else:
                         intobj = pyobj.c_ob_pypy_link
                         obj = llmemory.cast_int_to_adr(intobj)
@@ -524,7 +526,7 @@
         if pyobj.c_ob_pypy_link <> 0:
             if use_dict:
                 pyobject = llmemory.cast_ptr_to_adr(pyobj)
-                obj = self.refcnt_dict.get(pyobject)
+                obj = self.pypy_link_dict.get(pyobject)
             else:
                 intobj = pyobj.c_ob_pypy_link
                 obj = llmemory.cast_int_to_adr(intobj)
@@ -584,7 +586,7 @@
             pyobj.c_ob_refcnt += self.refcnt_add
             if self.refcnt_add > 0:
                 pyobject = llmemory.cast_ptr_to_adr(pyobj)
-                obj = self.refcnt_dict.get(pyobject)
+                obj = self.pypy_link_dict.get(pyobject)
                 self.gc.objects_to_trace.append(obj)
                 self.gc.visit_all_objects()
 
@@ -602,6 +604,13 @@
         else:
             self.tp_traverse(pyobj, self._visit_action, None)
 
+    def _pyobj_gc_refcnt_set(self, pygchdr, refcnt):
+        pygchdr.c_gc_refs &= self.RAWREFCOUNT_REFS_MASK_FINALIZED
+        pygchdr.c_gc_refs |= refcnt << self.RAWREFCOUNT_REFS_SHIFT
+
+    def _pyobj_gc_refcnt_get(self, pygchdr):
+        return pygchdr.c_gc_refs >> self.RAWREFCOUNT_REFS_SHIFT
+
     # --- Helpers ---
 
     def _gc_list_new(self):
diff --git a/rpython/memory/gc/rrc/incmark.py b/rpython/memory/gc/rrc/incmark.py
--- a/rpython/memory/gc/rrc/incmark.py
+++ b/rpython/memory/gc/rrc/incmark.py
@@ -110,7 +110,8 @@
             pyobj = llmemory.cast_adr_to_ptr(snapobj.pyobj, self.PYOBJ_HDR_PTR)
             pygchdr = self.pyobj_as_gc(pyobj)
             if (pygchdr != lltype.nullptr(self.PYOBJ_GC_HDR) and
-                    pygchdr.c_gc_refs != self.RAWREFCOUNT_REFS_UNTRACKED):
+                    pygchdr.c_gc_refs != self.RAWREFCOUNT_REFS_UNTRACKED and
+                    pygchdr.c_gc_refs != self.RAWREFCOUNT_REFS_REACHABLE):
                 break  # only look for non-gc
             addr = llmemory.cast_int_to_adr(snapobj.pypy_link)
             if (self.gc.header(addr).tid &
@@ -135,14 +136,16 @@
         pygchdr = self.pyobj_list.c_gc_next
         while pygchdr <> self.pyobj_list and consistent:
             next_old = pygchdr.c_gc_next
-            if pygchdr.c_gc_refs > 0:  # object is in snapshot
+            if pygchdr.c_gc_refs != self.RAWREFCOUNT_REFS_REACHABLE and \
+                    pygchdr.c_gc_refs != self.RAWREFCOUNT_REFS_UNTRACKED:  # object is in snapshot
                 consistent = self._check_consistency_gc(pygchdr,
                                                         self.pyobj_old_list)
                 if not consistent:
                     break
             else:
                 # new object, keep alive
-                pygchdr.c_gc_refs = 1 << self.RAWREFCOUNT_REFS_SHIFT
+                self._pyobj_gc_refcnt_set(pygchdr, 1)
+                #pygchdr.c_gc_refs = 1 << self.RAWREFCOUNT_REFS_SHIFT
                 pyobj = self.gc_as_pyobj(pygchdr)
                 if pyobj.c_ob_pypy_link != 0:
                     addr = llmemory.cast_int_to_adr(pyobj.c_ob_pypy_link)
@@ -175,12 +178,14 @@
             # continue previous loop, keep objects alive
             pygchdr = pygchdr_continue_gc
             while pygchdr <> self.pyobj_list:
-                pygchdr.c_gc_refs = 1 << self.RAWREFCOUNT_REFS_SHIFT
+                self._pyobj_gc_refcnt_set(pygchdr, 1)
+                #pygchdr.c_gc_refs = 1 << self.RAWREFCOUNT_REFS_SHIFT
                 pygchdr = pygchdr.c_gc_next
             pygchdr = self.pyobj_old_list.c_gc_next
             # resurrect "dead" objects
             while pygchdr <> self.pyobj_old_list:
-                pygchdr.c_gc_refs = 1 << self.RAWREFCOUNT_REFS_SHIFT
+                self._pyobj_gc_refcnt_set(pygchdr, 1)
+                #pygchdr.c_gc_refs = 1 << self.RAWREFCOUNT_REFS_SHIFT
                 pygchdr = pygchdr.c_gc_next
             # merge lists
             if not self._gc_list_is_empty(self.pyobj_old_list):
@@ -197,11 +202,13 @@
             # continue previous loop, keep objects alive
             pygchdr = pygchdr_continue_isolate
             while pygchdr <> self.pyobj_isolate_old_list:
-                pygchdr.c_gc_refs = 1 << self.RAWREFCOUNT_REFS_SHIFT
+                self._pyobj_gc_refcnt_set(pygchdr, 1)
+                #pygchdr.c_gc_refs = 1 << self.RAWREFCOUNT_REFS_SHIFT
                 pygchdr = pygchdr.c_gc_next
             # resurrect "dead" objects
             while pygchdr <> self.pyobj_isolate_dead_list:
-                pygchdr.c_gc_refs = 1 << self.RAWREFCOUNT_REFS_SHIFT
+                self._pyobj_gc_refcnt_set(pygchdr, 1)
+                #pygchdr.c_gc_refs = 1 << self.RAWREFCOUNT_REFS_SHIFT
                 pygchdr = pygchdr.c_gc_next
             # merge lists
             if not self._gc_list_is_empty(self.pyobj_isolate_old_list):
@@ -212,8 +219,11 @@
                                     self.pyobj_list)
 
     def _check_consistency_gc(self, pygchdr, pylist_dead_target):
-        snapobj = self.snapshot_objs[pygchdr.c_gc_refs - 1]
-        pygchdr.c_gc_refs = snapobj.refcnt
+        c_gc_refs = self._pyobj_gc_refcnt_get(pygchdr)
+        snapobj = self.snapshot_objs[c_gc_refs - 1]
+        #snapobj = self.snapshot_objs[pygchdr.c_gc_refs - 1]
+        self._pyobj_gc_refcnt_set(pygchdr, snapobj.refcnt)
+        #pygchdr.c_gc_refs = snapobj.refcnt
         if snapobj.refcnt == 0:  # object considered dead
             # check consistency (dead subgraphs can never change):
             pyobj = self.gc_as_pyobj(pygchdr)
@@ -241,6 +251,7 @@
                         "refcnt", snapobj.refcnt,
                         "refcnt original", snapobj.refcnt_original,
                         "link", snapobj.pypy_link)
+        debug_stop("snap " + print_label)
 
     def _check_consistency_p_list_old(self, pyobject, foo):
         pyobj = llmemory.cast_adr_to_ptr(pyobject, self.PYOBJ_HDR_PTR)
@@ -262,6 +273,7 @@
         elif refcnt >= REFCNT_FROM_PYPY:
             refcnt -= REFCNT_FROM_PYPY
         pyobj.c_ob_refcnt = refcnt
+        pyobj.c_ob_pypy_link = 0
 
     def _collect_roots(self):
         # Subtract all internal refcounts from the cyclic refcount
@@ -345,9 +357,11 @@
                                        self.GCFLAG_NO_HEAP_PTRS):
             pygchdr = self.pyobj_as_gc(pyobj)
             if (pygchdr != lltype.nullptr(self.PYOBJ_GC_HDR) and
-                    pygchdr.c_gc_refs > 0 and
-                    pygchdr.c_gc_refs != self.RAWREFCOUNT_REFS_UNTRACKED):
-                index = pygchdr.c_gc_refs - 1
+                    #c_gc_refs > 0 and
+                    pygchdr.c_gc_refs != self.RAWREFCOUNT_REFS_UNTRACKED and
+                    pygchdr.c_gc_refs != self.RAWREFCOUNT_REFS_REACHABLE):
+                c_gc_refs = self._pyobj_gc_refcnt_get(pygchdr)
+                index = c_gc_refs - 1
                 snapobj = self.snapshot_objs[index]
                 if snapobj.refcnt == 0:
                     addr = llmemory.cast_ptr_to_adr(snapobj)
@@ -388,10 +402,15 @@
         self.total_objs = total_objs
         self.objs_index = 0
         self.refs_index = 0
+        debug_print("take snapshot, count:", self.total_objs)
 
         # take snapshot of p_list_old
         self.p_list_old.foreach(self._take_snapshot_pyobject, None)
 
+        # set pypy_link for o_list_old to zero, in case they are encountered
+        # during tp_traverse (so they are excluded from the snapshot)
+        self.o_list_old.foreach(self._take_snapshot_o_clearlink, None)
+
         # take snapshot of gc objs
         pygchdr = self.pyobj_list.c_gc_next
         while pygchdr <> self.pyobj_list:
@@ -405,22 +424,32 @@
             pygchdr = pygchdr.c_gc_next
 
         # fix references
+        debug_print("fix references, count:", self.refs_index)
         for i in range(0, self.refs_index):
             addr = self.snapshot_refs[i]
             pyobj = llmemory.cast_adr_to_ptr(addr, self.PYOBJ_HDR_PTR)
             pygchdr = self.pyobj_as_gc(pyobj)
-            if (pygchdr <> lltype.nullptr(self.PYOBJ_GC_HDR) and
-                pygchdr.c_gc_refs != self.RAWREFCOUNT_REFS_UNTRACKED):
-                if pygchdr.c_gc_refs > 0:
-                    obj = self.snapshot_objs[pygchdr.c_gc_refs - 1]
+            if pygchdr != lltype.nullptr(self.PYOBJ_GC_HDR):
+                if (#c_gc_refs > 0 and
+                        pygchdr.c_gc_refs != self.RAWREFCOUNT_REFS_UNTRACKED and
+                        pygchdr.c_gc_refs != self.RAWREFCOUNT_REFS_REACHABLE):
+                    c_gc_refs = self._pyobj_gc_refcnt_get(pygchdr)
+                    obj = self.snapshot_objs[c_gc_refs - 1]
+                    debug_print("fix reference", i, "from", obj, "gc",
+                                pygchdr.c_gc_refs)
                 else:
                     obj = lltype.nullptr(self.PYOBJ_SNAPSHOT_OBJ)
             else:
                 obj = self.snapshot_objs[pyobj.c_ob_pypy_link - 1]
+                debug_print("fix reference", i, "from", obj, "non-gc",
+                            pyobj.c_ob_pypy_link - 1)
             self.snapshot_refs[i] = llmemory.cast_ptr_to_adr(obj)
 
-        # fix links of p_list_old back
+        # fix links of p_list_old and o_list_old back
         self.p_list_old.foreach(self._take_snapshot_fixlink, None)
+        self.pypy_link_dict.foreach(self._take_snapshot_o_fixlink, None)
+        self.pypy_link_dict.delete()
+        self.pypy_link_dict = self.gc.AddressDict()
 
     def _take_snapshot_count_gc(self, pygchdr):
         from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY
@@ -448,7 +477,9 @@
             if self.gc.header(addr).tid & (self.GCFLAG_VISITED |
                                            self.GCFLAG_NO_HEAP_PTRS):
                 refcnt += 1
-        pygchdr.c_gc_refs = self.objs_index + 1
+        debug_print("take snapshot", self.objs_index, "gc", pyobj)
+        self._pyobj_gc_refcnt_set(pygchdr, self.objs_index + 1)
+        #pygchdr.c_gc_refs = self.objs_index + 1
         obj = self.snapshot_objs[self.objs_index]
         obj.pyobj = llmemory.cast_ptr_to_adr(pyobj)
         obj.status = 1
@@ -469,8 +500,7 @@
         pyobj = self._pyobj(pyobject)
         pygchdr = self.pyobj_as_gc(pyobj)
         # only include non-gc
-        if (pygchdr == lltype.nullptr(self.PYOBJ_GC_HDR) or
-             pygchdr.c_gc_refs == self.RAWREFCOUNT_REFS_UNTRACKED):
+        if pygchdr == lltype.nullptr(self.PYOBJ_GC_HDR):
             self.p_list_count += 1
             refcnt = pyobj.c_ob_refcnt
             if refcnt >= REFCNT_FROM_PYPY_LIGHT:
@@ -486,8 +516,7 @@
         pyobj = self._pyobj(pyobject)
         pygchdr = self.pyobj_as_gc(pyobj)
         # only include non-gc
-        if (pygchdr == lltype.nullptr(self.PYOBJ_GC_HDR) or
-             pygchdr.c_gc_refs == self.RAWREFCOUNT_REFS_UNTRACKED):
+        if pygchdr == lltype.nullptr(self.PYOBJ_GC_HDR):
             refcnt = pyobj.c_ob_refcnt
             if refcnt >= REFCNT_FROM_PYPY_LIGHT:
                 refcnt -= REFCNT_FROM_PYPY_LIGHT
@@ -498,6 +527,7 @@
                 if self.gc.header(addr).tid & (self.GCFLAG_VISITED |
                                                self.GCFLAG_NO_HEAP_PTRS):
                     refcnt += 1
+            debug_print("take snapshot", self.objs_index, "non-gc", pyobj)
             obj = self.snapshot_objs[self.objs_index]
             obj.pyobj = llmemory.cast_ptr_to_adr(pyobj)
             obj.status = 1
@@ -513,12 +543,25 @@
         pyobj = self._pyobj(pyobject)
         pygchdr = self.pyobj_as_gc(pyobj)
         # only include non-gc
-        if (pygchdr == lltype.nullptr(self.PYOBJ_GC_HDR) or
-             pygchdr.c_gc_refs == self.RAWREFCOUNT_REFS_UNTRACKED):
+        if pygchdr == lltype.nullptr(self.PYOBJ_GC_HDR):
             obj_index = pyobj.c_ob_pypy_link - 1
             obj = self.snapshot_objs[obj_index]
             pyobj.c_ob_pypy_link = obj.pypy_link
 
+    def _take_snapshot_o_clearlink(self, pyobject, foo):
+        pyobj = self._pyobj(pyobject)
+        pygchdr = self.pyobj_as_gc(pyobj)
+        # only for non-gc
+        if pygchdr == lltype.nullptr(self.PYOBJ_GC_HDR):
+            link = llmemory.cast_int_to_adr(pyobj.c_ob_pypy_link)
+            self.pypy_link_dict.setitem(pyobject, link)
+            pyobj.c_ob_pypy_link = 0
+
+    def _take_snapshot_o_fixlink(self, pyobject, link, foo):
+        pyobj = self._pyobj(pyobject)
+        link_int = llmemory.cast_adr_to_int(link, "symbolic")
+        pyobj.c_ob_pypy_link = link_int
+
     def _take_snapshot_visit(pyobj, self_ptr):
         from rpython.rtyper.annlowlevel import cast_adr_to_nongc_instance
         #
@@ -531,9 +574,12 @@
         pygchdr = self.pyobj_as_gc(pyobj)
         curr = self.snapshot_curr
         index = curr.refs_index + curr.refs_len
-        if ((pygchdr <> lltype.nullptr(self.PYOBJ_GC_HDR) and
+        if ((pygchdr != lltype.nullptr(self.PYOBJ_GC_HDR) and
              pygchdr.c_gc_refs != self.RAWREFCOUNT_REFS_UNTRACKED) or
-                pyobj.c_ob_pypy_link != 0):
+                (pygchdr == lltype.nullptr(self.PYOBJ_GC_HDR) and
+                pyobj.c_ob_pypy_link != 0)):
+            debug_print("take ref", index, "curr refs_index", curr.refs_index,
+                        "curr refs_len", curr.refs_len, "whatever", pyobj)
             self.snapshot_refs[index] = llmemory.cast_ptr_to_adr(pyobj)
             curr.refs_len += 1
 
@@ -561,7 +607,8 @@
     def _check_snapshot_visit_action(self, pyobj, ignore):
         pygchdr = self.pyobj_as_gc(pyobj)
         if pygchdr <> lltype.nullptr(self.PYOBJ_GC_HDR) and \
-                pygchdr.c_gc_refs != self.RAWREFCOUNT_REFS_UNTRACKED:
+                pygchdr.c_gc_refs != self.RAWREFCOUNT_REFS_UNTRACKED and \
+                    pygchdr.c_gc_refs != self.RAWREFCOUNT_REFS_REACHABLE:
             # check consistency with snapshot
             curr = self.snapshot_curr
             curr_index = self.snapshot_curr_index
diff --git a/rpython/memory/gc/rrc/mark.py b/rpython/memory/gc/rrc/mark.py
--- a/rpython/memory/gc/rrc/mark.py
+++ b/rpython/memory/gc/rrc/mark.py
@@ -56,9 +56,9 @@
             self._debug_check_consistency(print_label="end-mark")
 
             # fix refcnt back
-            self.refcnt_dict.foreach(self._fix_refcnt_back, None)
-            self.refcnt_dict.delete()
-            self.refcnt_dict = self.gc.AddressDict()
+            self.pypy_link_dict.foreach(self._fix_refcnt_back, None)
+            self.pypy_link_dict.delete()
+            self.pypy_link_dict = self.gc.AddressDict()
             self.use_refcntdict = False
         else:
             self.p_list_old.foreach(self._major_trace, (False, False))
@@ -69,7 +69,7 @@
 
     def to_obj(self, pyobject):
         if self.use_refcntdict:
-            obj = self.refcnt_dict.get(pyobject)
+            obj = self.pypy_link_dict.get(pyobject)
         else:
             obj = llmemory.cast_int_to_adr(
                 self._pyobj(pyobject).c_ob_pypy_link)
@@ -127,19 +127,15 @@
             self._traverse(pyobj, -1)
             pygchdr = pygchdr.c_gc_next
 
-    def _pyobj_gc_refcnt_set(self, pygchdr, refcnt):
-        pygchdr.c_gc_refs &= self.RAWREFCOUNT_REFS_MASK_FINALIZED
-        pygchdr.c_gc_refs |= refcnt << self.RAWREFCOUNT_REFS_SHIFT
-
     def _obj_save_refcnt(self, pyobject, ignore):
         pyobj = self._pyobj(pyobject)
         link = llmemory.cast_int_to_adr(pyobj.c_ob_pypy_link)
-        self.refcnt_dict.setitem(pyobject, link)
+        self.pypy_link_dict.setitem(pyobject, link)
         pyobj.c_ob_pypy_link = pyobj.c_ob_refcnt
 
     def _obj_fix_refcnt(self, pyobject, ignore):
         pyobj = self._pyobj(pyobject)
-        obj = self.refcnt_dict.get(pyobject)
+        obj = self.pypy_link_dict.get(pyobject)
         gchdr = self.pyobj_as_gc(pyobj)
         if gchdr <> lltype.nullptr(self.PYOBJ_GC_HDR):
             rc = gchdr.c_gc_refs
@@ -205,7 +201,7 @@
         obj = llmemory.NULL
         if pyobj.c_ob_pypy_link <> 0:
             pyobject = llmemory.cast_ptr_to_adr(pyobj)
-            obj = self.refcnt_dict.get(pyobject)
+            obj = self.pypy_link_dict.get(pyobject)
             if not alive and self.gc.header(obj).tid & (
                     self.GCFLAG_VISITED | self.GCFLAG_NO_HEAP_PTRS):
                 # add fake refcount, to mark it as live
@@ -227,7 +223,7 @@
 
     def _mark_rawrefcount_linked(self, pyobject, ignore):
         pyobj = self._pyobj(pyobject)
-        obj = self.refcnt_dict.get(pyobject)
+        obj = self.pypy_link_dict.get(pyobject)
         if self.gc.header(obj).tid & (self.GCFLAG_VISITED |
                                       self.GCFLAG_NO_HEAP_PTRS):
             gchdr = self.pyobj_as_gc(pyobj)
diff --git a/rpython/memory/gc/test/dot/keep_cpython_inc_1.dot b/rpython/memory/gc/test/dot/keep_cpython_inc_4.dot
copy from rpython/memory/gc/test/dot/keep_cpython_inc_1.dot
copy to rpython/memory/gc/test/dot/keep_cpython_inc_4.dot
--- a/rpython/memory/gc/test/dot/keep_cpython_inc_1.dot
+++ b/rpython/memory/gc/test/dot/keep_cpython_inc_4.dot
@@ -1,18 +1,9 @@
 digraph G {
-    "a" [type=B, alive=y, ext_refcnt=1];
-    "b" [type=P, alive=n];
-    "c" [type=B, alive=y];
-    "d" [type=P, alive=y];
-    "e" [type=B, alive=y];
-    "f" [type=C, alive=y];
-    "g" [type=C, alive=y];
-    "h" [type=C, alive=y, ext_refcnt=1];
-    "a" -> "b" [removed=after_snap];
-    "b"-> "c";
-    "c" -> "d";
-    "c" -> "f";
-    "f" -> "c";
-    "d" -> "e";
-    "e" -> "g";
-    "h" -> "f" [added=after_snap];
+    "a" [type=C, alive=n];
+    "b" [type=C, alive=n];
+    "c" [type=C, alive=n];
+    "d" [type=B, alive=y, added=after_snap, rooted=y];
+    "a" -> "b";
+    "b" -> "c";
+    "c" -> "a";
 }
diff --git a/rpython/memory/gc/test/test_rawrefcount.py b/rpython/memory/gc/test/test_rawrefcount.py
--- a/rpython/memory/gc/test/test_rawrefcount.py
+++ b/rpython/memory/gc/test/test_rawrefcount.py
@@ -17,6 +17,7 @@
 RAWREFCOUNT_FINALIZER_LEGACY = RawRefCountBaseGC.RAWREFCOUNT_FINALIZER_LEGACY
 RAWREFCOUNT_FINALIZER_NONE = RawRefCountBaseGC.RAWREFCOUNT_FINALIZER_NONE
 RAWREFCOUNT_REFS_UNTRACKED = RawRefCountBaseGC.RAWREFCOUNT_REFS_UNTRACKED
+RAWREFCOUNT_REFS_REACHABLE = RawRefCountBaseGC.RAWREFCOUNT_REFS_REACHABLE
 
 S = lltype.GcForwardReference()
 S.become(lltype.GcStruct('S',
@@ -249,7 +250,7 @@
                              immortal=True)
         self.gcobjs.append(r1gc)
         if tracked:
-            r1gc.c_gc_refs = 0
+            r1gc.c_gc_refs = RAWREFCOUNT_REFS_REACHABLE
             if tuple:
                 r1gc.c_gc_next = self.tuple_list
                 r1gc.c_gc_prev = self.tuple_list.c_gc_prev
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -514,6 +514,9 @@
             self.rawrefcount_to_obj_ptr = getfn(
                 GCClass.rawrefcount_to_obj, [s_gc, SomeAddress()], s_gcref,
                 inline = True)
+            self.rawrefcount_check_state_ptr = getfn(
+                GCClass.rawrefcount_check_state, [s_gc], annmodel.SomeBool(),
+                inline = True)
             self.rawrefcount_next_dead_ptr = getfn(
                 GCClass.rawrefcount_next_dead, [s_gc], SomeAddress(),
                 inline = True)
@@ -1424,6 +1427,11 @@
                   [self.rawrefcount_to_obj_ptr, self.c_const_gc, v_pyobject],
                   resultvar=hop.spaceop.result)
 
+    def gct_gc_rawrefcount_check_state(self, hop):
+        hop.genop("direct_call",
+                  [self.rawrefcount_check_state_ptr, self.c_const_gc],
+                  resultvar=hop.spaceop.result)
+
     def gct_gc_rawrefcount_next_dead(self, hop):
         assert hop.spaceop.result.concretetype == llmemory.Address
         hop.genop("direct_call",
diff --git a/rpython/rlib/rawrefcount.py b/rpython/rlib/rawrefcount.py
--- a/rpython/rlib/rawrefcount.py
+++ b/rpython/rlib/rawrefcount.py
@@ -117,6 +117,10 @@
     return p
 
 @not_rpython
+def check_state():
+    return True
+
+ at not_rpython
 def next_dead(OB_PTR_TYPE):
     """When the GC runs, it finds some pyobjs to be dead
     but cannot immediately dispose of them (it doesn't know how to call
@@ -387,6 +391,19 @@
                         resulttype = llmemory.GCREF)
         return _spec_p(hop, v_p)
 
+
+class Entry(ExtRegistryEntry):
+    _about_ = check_state
+
+    def compute_result_annotation(self):
+        from rpython.annotator import model as annmodel
+        return annmodel.SomeBool()
+
+    def specialize_call(self, hop):
+        hop.exception_cannot_occur()
+        return hop.genop('gc_rawrefcount_check_state', [],
+                         resulttype = hop.r_result)
+
 class Entry(ExtRegistryEntry):
     _about_ = (next_dead, cyclic_garbage_head, next_cyclic_isolate,
                next_garbage_pyobj)
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -984,6 +984,9 @@
     def op_gc_rawrefcount_mark_deallocating(self, *args):
         raise NotImplementedError("gc_rawrefcount_mark_deallocating")
 
+    def op_gc_rawrefcount_check_state(self, *args):
+        raise NotImplementedError("gc_rawrefcount_check_state")
+
     def op_gc_rawrefcount_next_dead(self, *args):
         raise NotImplementedError("gc_rawrefcount_next_dead")
 
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -528,6 +528,7 @@
     'gc_rawrefcount_mark_deallocating':     LLOp(),
     'gc_rawrefcount_from_obj':              LLOp(sideeffects=False),
     'gc_rawrefcount_to_obj':                LLOp(sideeffects=False),
+    'gc_rawrefcount_check_state':           LLOp(sideeffects=False),
     'gc_rawrefcount_next_dead':             LLOp(),
     'gc_rawrefcount_next_cyclic_isolate':   LLOp(),
     'gc_rawrefcount_cyclic_garbage_head':   LLOp(sideeffects=False),

From pypy.commits at gmail.com  Thu Oct 10 15:34:58 2019
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 10 Oct 2019 12:34:58 -0700 (PDT)
Subject: [pypy-commit] pypy default: Stop updating hypothesis: 4.40 is
 incompatible with our ancient pytest
Message-ID: <5d9f87e2.1c69fb81.45ee8.25c7@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r97757:9b953c97479f
Date: 2019-10-10 20:34 +0100
http://bitbucket.org/pypy/pypy/changeset/9b953c97479f/

Log:	Stop updating hypothesis: 4.40 is incompatible with our ancient
	pytest

diff --git a/requirements.txt b/requirements.txt
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,5 +6,5 @@
 vmprof>=0.4.10; 'x86' in platform.machine #skip arm, s390x
 
 # hypothesis is used for test generation on untranslated tests
-hypothesis
+hypothesis<4.40
 enum34>=1.1.2

From pypy.commits at gmail.com  Fri Oct 11 04:45:16 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 11 Oct 2019 01:45:16 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix translation, tweak output
Message-ID: <5da0411c.1c69fb81.a4ca9.ba66@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r97761:c866b476cfee
Date: 2019-10-11 10:43 +0200
http://bitbucket.org/pypy/pypy/changeset/c866b476cfee/

Log:	fix translation, tweak output

diff --git a/rpython/jit/metainterp/quasiimmut.py b/rpython/jit/metainterp/quasiimmut.py
--- a/rpython/jit/metainterp/quasiimmut.py
+++ b/rpython/jit/metainterp/quasiimmut.py
@@ -27,12 +27,14 @@
     return qmut
 
 def make_invalidation_function(STRUCT, mutatefieldname):
-    #
+    # fake a repr
+    descr_repr = "FieldDescr(%s, '%s')" % (STRUCT.TO, mutatefieldname)
+
     def _invalidate_now(p):
         qmut_ptr = getattr(p, mutatefieldname)
         setattr(p, mutatefieldname, lltype.nullptr(rclass.OBJECT))
         qmut = cast_base_ptr_to_instance(QuasiImmut, qmut_ptr)
-        qmut.invalidate(mutatefieldname)
+        qmut.invalidate(descr_repr)
     _invalidate_now._dont_inline_ = True
     #
     def invalidation(p):
@@ -46,7 +48,7 @@
     if qmut_ref:
         cpu.bh_setfield_gc_r(p, ConstPtr.value, mutatefielddescr)
         qmut = cast_gcref_to_instance(QuasiImmut, qmut_ref)
-        qmut.invalidate(mutatefielddescr.fieldname)
+        qmut.invalidate(mutatefielddescr.repr_of_descr())
 
 
 class QuasiImmut(object):
@@ -79,7 +81,7 @@
         # already invalidated; see below
         self.compress_limit = (len(self.looptokens_wrefs) + 15) * 2
 
-    def invalidate(self, fieldname=None):
+    def invalidate(self, descr_repr=None):
         debug_start("jit-invalidate-quasi-immutable")
         # When this is called, all the loops that we record become
         # invalid: all GUARD_NOT_INVALIDATED in these loops (and
@@ -104,7 +106,7 @@
                 if not we_are_translated():
                     self.cpu.stats.invalidated_token_numbers.add(
                         looptoken.number)
-        debug_print("fieldname", fieldname or "<unknown>", "invalidated", invalidated)
+        debug_print("fieldname", descr_repr or "<unknown>", "invalidated", invalidated)
         debug_stop("jit-invalidate-quasi-immutable")
 
 

From pypy.commits at gmail.com  Fri Oct 11 06:41:10 2019
From: pypy.commits at gmail.com (mattip)
Date: Fri, 11 Oct 2019 03:41:10 -0700 (PDT)
Subject: [pypy-commit] pypy default: refactor repackage script to generate
 exe
Message-ID: <5da05c46.1c69fb81.f0ef.5c33@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r97762:241777245f33
Date: 2019-10-11 13:40 +0300
http://bitbucket.org/pypy/pypy/changeset/241777245f33/

Log:	refactor repackage script to generate exe

diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,11 +1,15 @@
 # Edit these appropriately before running this script
 pmaj=2  # python main version: 2 or 3
 pmin=7  # python minor version
-exe=pypy # pypy3 or pypy
 maj=7
 min=2
-rev=0rc0
+rev=0rc2
 
+case $pmaj in
+    "2") exe=pypy;;
+    "3") exe=pypy3;;
+    *) echo invalid pmaj=$pmaj; exit 1;;
+esac
 
 branchname=release-pypy$pmaj.$pmin-v$maj.x # ==OR== release-v$maj.x  # ==OR== release-v$maj.$min.x
 tagname=release-pypy$pmaj.$pmin-v$maj.$min.$rev  # ==OR== release-$maj.$min

From pypy.commits at gmail.com  Fri Oct 11 07:30:39 2019
From: pypy.commits at gmail.com (mattip)
Date: Fri, 11 Oct 2019 04:30:39 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: merge default into py3.6
Message-ID: <5da067df.1c69fb81.c7a91.c08c@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r97763:f963c5cf8e2e
Date: 2019-10-11 14:29 +0300
http://bitbucket.org/pypy/pypy/changeset/f963c5cf8e2e/

Log:	merge default into py3.6

diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,15 +1,18 @@
 # Edit these appropriately before running this script
 pmaj=2  # python main version: 2 or 3
 pmin=7  # python minor version
-exe=pypy3 # pypy3 or pypy
 maj=7
 min=2
-rev=0
+rev=0rc2
 
+case $pmaj in
+    "2") exe=pypy;;
+    "3") exe=pypy3;;
+    *) echo invalid pmaj=$pmaj; exit 1;;
+esac
 
 branchname=release-pypy$pmaj.$pmin-v$maj.x # ==OR== release-v$maj.x  # ==OR== release-v$maj.$min.x
-tagname=release-candidate-pypy$pmaj.$pmin-v$maj.$min.$rev  # ==OR== release-$maj.$min
-# tagname=release-pypy$pmaj.$pmin-v$maj.$min.$rev  # ==OR== release-$maj.$min
+tagname=release-pypy$pmaj.$pmin-v$maj.$min.$rev  # ==OR== release-$maj.$min
 
 echo checking hg log -r $branchname
 hg log -r $branchname || exit 1
diff --git a/rpython/jit/metainterp/quasiimmut.py b/rpython/jit/metainterp/quasiimmut.py
--- a/rpython/jit/metainterp/quasiimmut.py
+++ b/rpython/jit/metainterp/quasiimmut.py
@@ -27,12 +27,14 @@
     return qmut
 
 def make_invalidation_function(STRUCT, mutatefieldname):
-    #
+    # fake a repr
+    descr_repr = "FieldDescr(%s, '%s')" % (STRUCT.TO, mutatefieldname)
+
     def _invalidate_now(p):
         qmut_ptr = getattr(p, mutatefieldname)
         setattr(p, mutatefieldname, lltype.nullptr(rclass.OBJECT))
         qmut = cast_base_ptr_to_instance(QuasiImmut, qmut_ptr)
-        qmut.invalidate(mutatefieldname)
+        qmut.invalidate(descr_repr)
     _invalidate_now._dont_inline_ = True
     #
     def invalidation(p):
@@ -46,7 +48,7 @@
     if qmut_ref:
         cpu.bh_setfield_gc_r(p, ConstPtr.value, mutatefielddescr)
         qmut = cast_gcref_to_instance(QuasiImmut, qmut_ref)
-        qmut.invalidate(mutatefielddescr.fieldname)
+        qmut.invalidate(mutatefielddescr.repr_of_descr())
 
 
 class QuasiImmut(object):
@@ -79,7 +81,7 @@
         # already invalidated; see below
         self.compress_limit = (len(self.looptokens_wrefs) + 15) * 2
 
-    def invalidate(self, fieldname=None):
+    def invalidate(self, descr_repr=None):
         debug_start("jit-invalidate-quasi-immutable")
         # When this is called, all the loops that we record become
         # invalid: all GUARD_NOT_INVALIDATED in these loops (and
@@ -104,7 +106,7 @@
                 if not we_are_translated():
                     self.cpu.stats.invalidated_token_numbers.add(
                         looptoken.number)
-        debug_print("fieldname", fieldname or "<unknown>", "invalidated", invalidated)
+        debug_print("fieldname", descr_repr or "<unknown>", "invalidated", invalidated)
         debug_stop("jit-invalidate-quasi-immutable")
 
 

From pypy.commits at gmail.com  Sat Oct 12 06:19:08 2019
From: pypy.commits at gmail.com (arigo)
Date: Sat, 12 Oct 2019 03:19:08 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: #3088: don't grow the lzma.decompress()
 buffer past max_length
Message-ID: <5da1a89c.1c69fb81.2086a.1c25@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.6
Changeset: r97764:466d2e12ff4b
Date: 2019-10-12 12:18 +0200
http://bitbucket.org/pypy/pypy/changeset/466d2e12ff4b/

Log:	#3088: don't grow the lzma.decompress() buffer past max_length

diff --git a/lib_pypy/_lzma.py b/lib_pypy/_lzma.py
--- a/lib_pypy/_lzma.py
+++ b/lib_pypy/_lzma.py
@@ -634,6 +634,8 @@
                         break
                     # ran out of space in the output buffer, let's grow it
                     bufsiz += (bufsiz >> 3) + 6
+                    if max_length > 0 and bufsiz > max_length:
+                        bufsiz = max_length
                     next_out = m.realloc(orig_out, bufsiz)
                     if next_out == ffi.NULL:
                         # realloc unsuccessful

From pypy.commits at gmail.com  Sat Oct 12 06:38:49 2019
From: pypy.commits at gmail.com (arigo)
Date: Sat, 12 Oct 2019 03:38:49 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: #3090: lzma sometimes fails to decompress
 a file
Message-ID: <5da1ad39.1c69fb81.f3095.73ca@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.6
Changeset: r97765:3a35a0ae4463
Date: 2019-10-12 12:38 +0200
http://bitbucket.org/pypy/pypy/changeset/3a35a0ae4463/

Log:	#3090: lzma sometimes fails to decompress a file

	Fixed by back-porting CPython commit
	a3c53a1b45b05bcb69660eac5a271443b37ecc42

diff --git a/lib_pypy/_lzma.py b/lib_pypy/_lzma.py
--- a/lib_pypy/_lzma.py
+++ b/lib_pypy/_lzma.py
@@ -583,8 +583,16 @@
                 self.clear_input_buffer()
             elif lzs.avail_in == 0:
                 # completed successfully!
-                self.needs_input = True
                 lzs.next_in = ffi.NULL
+                if lzs.avail_out == 0:
+                    # (avail_in==0 && avail_out==0)
+                    # Maybe lzs's internal state still have a few bytes can
+                    # be output, try to output them next time.
+                    self.needs_input = False
+                    assert max_length >= 0   # if < 0, lzs.avail_out always > 0
+                else:
+                    # Input buffer exhausted, output buffer has space.
+                    self.needs_input = True
                 self.clear_input_buffer()
             else:
                 self.needs_input = False
@@ -599,9 +607,6 @@
         lzs.next_in = buf
         lzs.avail_in = buf_len
 
-        if buf_len == 0:
-            return b""
-
         bufsiz = self._bufsiz
         if not (max_length < 0 or max_length > io.DEFAULT_BUFFER_SIZE):
             bufsiz = max_length
@@ -616,7 +621,8 @@
 
         try:
             while True:
-                ret = catch_lzma_error(m.lzma_code, lzs, m.LZMA_RUN)
+                ret = catch_lzma_error(m.lzma_code, lzs, m.LZMA_RUN,
+                    ignore_buf_error=(lzs.avail_in == 0 and lzs.avail_out > 0))
                 data_size = int(ffi.cast('uintptr_t', lzs.next_out)) - int(ffi.cast('uintptr_t', orig_out))
                 # data_size is the amount lzma_code has already outputted
 
@@ -626,10 +632,11 @@
                 if ret == m.LZMA_STREAM_END:
                     self.eof = True
                     break
-                elif lzs.avail_in == 0:
-                    # it ate everything
-                    break
                 elif lzs.avail_out == 0:
+                    # Need to check lzs->avail_out before lzs->avail_in.
+                    # Maybe lzs's internal state still have a few bytes
+                    # can be output, grow the output buffer and continue
+                    # if max_lengh < 0.
                     if data_size == max_length:
                         break
                     # ran out of space in the output buffer, let's grow it
@@ -647,6 +654,9 @@
 
                     lzs.next_out = orig_out + data_size
                     lzs.avail_out = bufsiz - data_size
+                elif lzs.avail_in == 0:
+                    # it ate everything
+                    break
 
             result = ffi.buffer(orig_out, data_size)[:]
         finally:

From pypy.commits at gmail.com  Sat Oct 12 18:10:23 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Sat, 12 Oct 2019 15:10:23 -0700 (PDT)
Subject: [pypy-commit] pypy fix-descrmismatch-crash: fix first level of
 problems: args.firstarg() is always None in this context, 
Message-ID: <5da24f4f.1c69fb81.ca3c7.f9de@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: fix-descrmismatch-crash
Changeset: r97766:da9a65e1debd
Date: 2019-10-13 00:09 +0200
http://bitbucket.org/pypy/pypy/changeset/da9a65e1debd/

Log:	fix first level of problems: args.firstarg() is always None in this
	context, use w_obj instead

diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -709,6 +709,7 @@
                     self.func__args__ = func
                 elif unwrap_spec == [self_type, ObjSpace, Arguments]:
                     self.__class__ = BuiltinCodePassThroughArguments1
+                    self.descr_reqcls = self_type
                     miniglobals = {'func': func, 'self_type': self_type}
                     d = {}
                     source = """if 1:
@@ -808,7 +809,7 @@
         try:
             w_result = self.func__args__(space, w_obj, args)
         except DescrMismatch:
-            return args.firstarg().descr_call_mismatch(space,
+            return w_obj.descr_call_mismatch(space,
                                                   self.descrmismatch_op,
                                                   self.descr_reqcls,
                                                   args.prepend(w_obj))
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -1005,3 +1005,13 @@
 
         d.update(**{clash: 33})
         dict.update(d, **{clash: 33})
+
+
+
+class AppTestFastPathCrash(object):
+    def test_fast_path_crash(self):
+        with raises(TypeError) as excinfo:
+            # does not crash in BuiltinCodePassThroughArguments0.funcrun
+            dict.__init__.im_func(0)
+        print(str(excinfo.value))
+        assert str(excinfo.value) == "'dict' object expected, got 'int' instead"

From pypy.commits at gmail.com  Sun Oct 13 04:59:50 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 13 Oct 2019 01:59:50 -0700 (PDT)
Subject: [pypy-commit] pypy fix-descrmismatch-crash: improve error message
Message-ID: <5da2e786.1c69fb81.4a165.fd00@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: fix-descrmismatch-crash
Changeset: r97767:5aa0e1eef0dd
Date: 2019-10-13 11:58 +0300
http://bitbucket.org/pypy/pypy/changeset/5aa0e1eef0dd/

Log:	improve error message

diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -1009,9 +1009,18 @@
 
 
 class AppTestFastPathCrash(object):
+    def setup_class(cls):
+        cls.w_runappdirect = cls.space.wrap(cls.runappdirect)
+
     def test_fast_path_crash(self):
-        with raises(TypeError) as excinfo:
-            # does not crash in BuiltinCodePassThroughArguments0.funcrun
-            dict.__init__.im_func(0)
-        print(str(excinfo.value))
-        assert str(excinfo.value) == "'dict' object expected, got 'int' instead"
+        # issue bb-3091 crash in BuiltinCodePassThroughArguments0.funcrun
+        for obj in (dict, set, frozenset):
+            with raises(TypeError) as excinfo:
+                if self.runappdirect:
+                    msg_fmt = "'%s' object but received a '%s'"
+                    obj.__init__(0)
+                else:
+                    msg_fmt = "'%s' object expected, got '%s'"
+                    obj.__init__.im_func(0)
+            msg = msg_fmt %(obj.__name__, 'int')
+            assert msg in str(excinfo.value)
diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -496,6 +496,12 @@
 
 
 class W_SetObject(W_BaseSetObject):
+
+    #overridden here so the error is reported correctly
+    def __init__(self, space, w_iterable=None):
+        """Initialize the set by taking ownership of 'setdata'."""
+        W_BaseSetObject.__init__(self, space, w_iterable)
+
     def _newobj(self, space, w_iterable):
         """Make a new set by taking ownership of 'w_iterable'."""
         if type(self) is W_SetObject:
@@ -516,7 +522,7 @@
 
 Build an unordered collection.""",
     __new__ = gateway.interp2app(W_SetObject.descr_new),
-    __init__ = gateway.interp2app(W_BaseSetObject.descr_init),
+    __init__ = gateway.interp2app(W_SetObject.descr_init),
     __repr__ = gateway.interp2app(W_BaseSetObject.descr_repr),
     __hash__ = None,
     __cmp__ = gateway.interp2app(W_BaseSetObject.descr_cmp),
@@ -572,6 +578,11 @@
 class W_FrozensetObject(W_BaseSetObject):
     hash = 0
 
+    #overridden here so the error is reported correctly
+    def __init__(self, space, w_iterable=None):
+        """Initialize the frozenset by taking ownership of 'setdata'."""
+        W_BaseSetObject.__init__(self, space, w_iterable)
+
     def _cleanup_(self):
         # in case there are frozenset objects existing during
         # translation, make sure we don't translate a cached hash
@@ -639,6 +650,7 @@
 
 Build an immutable unordered collection.""",
     __new__ = gateway.interp2app(W_FrozensetObject.descr_new2),
+    __init__ = gateway.interp2app(W_FrozensetObject.descr_init),
     __repr__ = gateway.interp2app(W_BaseSetObject.descr_repr),
     __hash__ = gateway.interp2app(W_FrozensetObject.descr_hash),
     __cmp__ = gateway.interp2app(W_BaseSetObject.descr_cmp),

From pypy.commits at gmail.com  Mon Oct 14 01:59:31 2019
From: pypy.commits at gmail.com (arigo)
Date: Sun, 13 Oct 2019 22:59:31 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: issue #3092
Message-ID: <5da40ec3.1c69fb81.26597.6b56@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.6
Changeset: r97768:234d0560480e
Date: 2019-10-14 07:58 +0200
http://bitbucket.org/pypy/pypy/changeset/234d0560480e/

Log:	issue #3092

	PyObject_GenericGetDict, PyObject_GenericSetDict

diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -308,3 +308,5 @@
             return 0
     return 1
 
+#def PyObject_GenericGetDict(space, w_obj, context):
+#    unlike CPython, you'll find this one in object.py together with ..SetDict
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -444,3 +444,16 @@
             del d[w_obj]
         except KeyError:
             pass
+
+ at cpython_api([PyObject, rffi.VOIDP], PyObject)
+def PyObject_GenericGetDict(space, w_obj, context):
+    from pypy.interpreter.typedef import descr_get_dict
+    return descr_get_dict(space, w_obj)
+
+ at cpython_api([PyObject, PyObject, rffi.VOIDP], rffi.INT_real, error=-1)
+def PyObject_GenericSetDict(space, w_obj, w_value, context):
+    from pypy.interpreter.typedef import descr_set_dict
+    if w_value is None:
+        raise oefmt(space.w_TypeError, "cannot delete __dict__")
+    descr_set_dict(space, w_obj, w_value)
+    return 0
diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py
--- a/pypy/module/cpyext/test/test_object.py
+++ b/pypy/module/cpyext/test/test_object.py
@@ -451,6 +451,31 @@
         assert n == 1
         module.leave(obj2)
 
+    def test_GenericGetSetDict(self):
+        module = self.import_extension('test_GenericGetSetDict', [
+            ('test1', 'METH_VARARGS',
+             """
+                 PyObject *obj = PyTuple_GET_ITEM(args, 0);
+                 PyObject *newdict = PyTuple_GET_ITEM(args, 1);
+
+                 PyObject *olddict = PyObject_GenericGetDict(obj, NULL);
+                 if (olddict == NULL)
+                    return NULL;
+                 int res = PyObject_GenericSetDict(obj, newdict, NULL);
+                 if (res != 0)
+                     return NULL;
+                 return olddict;
+             """)])
+        class A:
+            pass
+        a = A()
+        a.x = 42
+        nd = {'y': 43}
+        d = module.test1(a, nd)
+        assert d == {'x': 42}
+        assert a.y == 43
+        assert a.__dict__ is nd
+
 
 class AppTestPyBuffer_FillInfo(AppTestCpythonExtensionBase):
     """

From pypy.commits at gmail.com  Mon Oct 14 03:19:02 2019
From: pypy.commits at gmail.com (arigo)
Date: Mon, 14 Oct 2019 00:19:02 -0700 (PDT)
Subject: [pypy-commit] cffi default: Add a warning when we use in cdef() a
 global variable without also specifying a storage class (extern or static)
Message-ID: <5da42166.1c69fb81.a4845.d740@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3294:32b6ed4cb029
Date: 2019-10-14 09:16 +0200
http://bitbucket.org/cffi/cffi/changeset/32b6ed4cb029/

Log:	Add a warning when we use in cdef() a global variable without also
	specifying a storage class (extern or static)

diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -156,6 +156,13 @@
                           "confuse pre-parsing.")
             break
 
+def _warn_for_non_extern_non_static_global_variable(decl):
+    if not decl.storage:
+        import warnings
+        warnings.warn("Declaration of global variable '%s' in cdef() should "
+                      "be marked 'extern' for consistency (or possibly "
+                      "'static' in API mode)" % (decl.name,))
+
 def _preprocess(csource):
     # Remove comments.  NOTE: this only work because the cdef() section
     # should not contain any string literal!
@@ -506,6 +513,7 @@
                     if (quals & model.Q_CONST) and not tp.is_array_type:
                         self._declare('constant ' + decl.name, tp, quals=quals)
                     else:
+                        _warn_for_non_extern_non_static_global_variable(decl)
                         self._declare('variable ' + decl.name, tp, quals=quals)
 
     def parse_type(self, cdecl):
diff --git a/testing/cffi0/test_function.py b/testing/cffi0/test_function.py
--- a/testing/cffi0/test_function.py
+++ b/testing/cffi0/test_function.py
@@ -113,7 +113,7 @@
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
             int fputs(const char *, void *);
-            void *stderr;
+            extern void *stderr;
         """)
         needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
@@ -130,7 +130,7 @@
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
             int fputs(char *, void *);
-            void *stderr;
+            extern void *stderr;
         """)
         needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
@@ -147,7 +147,7 @@
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
            int fprintf(void *, const char *format, ...);
-           void *stderr;
+           extern void *stderr;
         """)
         needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
@@ -209,7 +209,7 @@
             py.test.skip("probably no symbol 'stderr' in the lib")
         ffi.cdef("""
             int fputs(const char *, void *);
-            void *stderr;
+            extern void *stderr;
         """)
         needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
@@ -256,7 +256,7 @@
             py.test.skip("probably no symbol 'stdout' in the lib")
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
-            void *stdout;
+            extern void *stdout;
         """)
         needs_dlopen_none()
         C = ffi.dlopen(None)
@@ -496,7 +496,7 @@
         ffi.cdef("""
             typedef enum { MYE1, MYE2 } myenum_t;
             double myfunc(double);
-            double myvar;
+            extern double myvar;
             const double myconst;
             #define MYFOO 42
         """)
@@ -507,7 +507,7 @@
         if self.Backend is CTypesBackend:
             py.test.skip("not with the ctypes backend")
         ffi = FFI(backend=self.Backend())
-        ffi.cdef("int foobar(void); int foobaz;")
+        ffi.cdef("int foobar(void); extern int foobaz;")
         lib = ffi.dlopen(lib_m)
         ffi.dlclose(lib)
         e = py.test.raises(ValueError, getattr, lib, 'foobar')
diff --git a/testing/cffi0/test_ownlib.py b/testing/cffi0/test_ownlib.py
--- a/testing/cffi0/test_ownlib.py
+++ b/testing/cffi0/test_ownlib.py
@@ -201,7 +201,7 @@
             py.test.skip("fix the auto-generation of the tiny test lib")
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
-            int my_array[7];
+            extern int my_array[7];
         """)
         ownlib = ffi.dlopen(self.module)
         for i in range(7):
@@ -223,7 +223,7 @@
             py.test.skip("not supported by the ctypes backend")
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
-            int my_array[];
+            extern int my_array[];
         """)
         ownlib = ffi.dlopen(self.module)
         for i in range(7):
@@ -291,7 +291,7 @@
                 long bottom;
             } RECT;
             
-            long left, top, right, bottom;
+            extern long left, top, right, bottom;
 
             RECT ReturnRect(int i, RECT ar, RECT* br, POINT cp, RECT dr,
                         RECT *er, POINT fp, RECT gr);
@@ -321,7 +321,7 @@
         if self.Backend is CTypesBackend:
             py.test.skip("not implemented with the ctypes backend")
         ffi = FFI(backend=self.Backend())
-        ffi.cdef("long left; int test_getting_errno(void);")
+        ffi.cdef("extern long left; int test_getting_errno(void);")
         lib = ffi.dlopen(self.module)
         lib.left = 123456
         p = ffi.addressof(lib, "left")
diff --git a/testing/cffi0/test_parsing.py b/testing/cffi0/test_parsing.py
--- a/testing/cffi0/test_parsing.py
+++ b/testing/cffi0/test_parsing.py
@@ -324,6 +324,7 @@
     assert value == sys.maxsize * 2 - 40
 
 def test__is_constant_globalvar():
+    import warnings
     for input, expected_output in [
         ("int a;",          False),
         ("const int a;",    True),
@@ -341,10 +342,13 @@
         ("const int a[5][6];", False),
         ]:
         ffi = FFI()
-        ffi.cdef(input)
+        with warnings.catch_warnings(record=True) as log:
+            warnings.simplefilter("always")
+            ffi.cdef(input)
         declarations = ffi._parser._declarations
         assert ('constant a' in declarations) == expected_output
         assert ('variable a' in declarations) == (not expected_output)
+        assert len(log) == (1 - expected_output)
 
 def test_restrict():
     from cffi import model
@@ -354,7 +358,7 @@
         ("int *a;",            False),
         ]:
         ffi = FFI()
-        ffi.cdef(input)
+        ffi.cdef("extern " + input)
         tp, quals = ffi._parser._declarations['variable a']
         assert bool(quals & model.Q_RESTRICT) == expected_output
 
diff --git a/testing/cffi0/test_verify.py b/testing/cffi0/test_verify.py
--- a/testing/cffi0/test_verify.py
+++ b/testing/cffi0/test_verify.py
@@ -286,7 +286,7 @@
 def test_var_signed_integer_types():
     ffi = FFI()
     lst = all_signed_integer_types(ffi)
-    csource = "\n".join(["%s somevar_%s;" % (tp, tp.replace(' ', '_'))
+    csource = "\n".join(["static %s somevar_%s;" % (tp, tp.replace(' ', '_'))
                          for tp in lst])
     ffi.cdef(csource)
     lib = ffi.verify(csource)
@@ -305,7 +305,7 @@
 def test_var_unsigned_integer_types():
     ffi = FFI()
     lst = all_unsigned_integer_types(ffi)
-    csource = "\n".join(["%s somevar_%s;" % (tp, tp.replace(' ', '_'))
+    csource = "\n".join(["static %s somevar_%s;" % (tp, tp.replace(' ', '_'))
                          for tp in lst])
     ffi.cdef(csource)
     lib = ffi.verify(csource)
@@ -817,8 +817,8 @@
 
 def test_access_variable():
     ffi = FFI()
-    ffi.cdef("int foo(void);\n"
-             "int somenumber;")
+    ffi.cdef("static int foo(void);\n"
+             "static int somenumber;")
     lib = ffi.verify("""
         static int somenumber = 2;
         static int foo(void) {
@@ -835,7 +835,7 @@
 def test_access_address_of_variable():
     # access the address of 'somenumber': need a trick
     ffi = FFI()
-    ffi.cdef("int somenumber; static int *const somenumberptr;")
+    ffi.cdef("static int somenumber; static int *const somenumberptr;")
     lib = ffi.verify("""
         static int somenumber = 2;
         #define somenumberptr (&somenumber)
@@ -848,7 +848,7 @@
 def test_access_array_variable(length=5):
     ffi = FFI()
     ffi.cdef("int foo(int);\n"
-             "int somenumber[%s];" % (length,))
+             "static int somenumber[%s];" % (length,))
     lib = ffi.verify("""
         static int somenumber[] = {2, 2, 3, 4, 5};
         static int foo(int i) {
@@ -880,7 +880,7 @@
     ffi = FFI()
     ffi.cdef("struct foo { int x; ...; };\n"
              "int foo(int);\n"
-             "struct foo stuff;")
+             "static struct foo stuff;")
     lib = ffi.verify("""
         struct foo { int x, y, z; };
         static struct foo stuff = {2, 5, 8};
@@ -904,9 +904,9 @@
 
 def test_access_callback():
     ffi = FFI()
-    ffi.cdef("int (*cb)(int);\n"
-             "int foo(int);\n"
-             "void reset_cb(void);")
+    ffi.cdef("static int (*cb)(int);\n"
+             "static int foo(int);\n"
+             "static void reset_cb(void);")
     lib = ffi.verify("""
         static int g(int x) { return x * 7; }
         static int (*cb)(int);
@@ -922,9 +922,9 @@
 def test_access_callback_function_typedef():
     ffi = FFI()
     ffi.cdef("typedef int mycallback_t(int);\n"
-             "mycallback_t *cb;\n"
-             "int foo(int);\n"
-             "void reset_cb(void);")
+             "static mycallback_t *cb;\n"
+             "static int foo(int);\n"
+             "static void reset_cb(void);")
     lib = ffi.verify("""
         static int g(int x) { return x * 7; }
         static int (*cb)(int);
@@ -1074,7 +1074,7 @@
 def test_autofilled_struct_as_argument_dynamic():
     ffi = FFI()
     ffi.cdef("struct foo_s { long a; ...; };\n"
-             "int (*foo)(struct foo_s);")
+             "static int (*foo)(struct foo_s);")
     lib = ffi.verify("""
         struct foo_s {
             double b;
@@ -1083,7 +1083,7 @@
         int foo1(struct foo_s s) {
             return (int)s.a - (int)s.b;
         }
-        int (*foo)(struct foo_s s) = &foo1;
+        static int (*foo)(struct foo_s s) = &foo1;
     """)
     e = py.test.raises(NotImplementedError, lib.foo, "?")
     msg = ("ctype 'struct foo_s' not supported as argument.  It is a struct "
@@ -1453,7 +1453,7 @@
         py.test.skip("_Bool not in MSVC")
     ffi = FFI()
     ffi.cdef("struct foo_s { _Bool x; };"
-             "_Bool foo(_Bool); _Bool (*foop)(_Bool);")
+             "_Bool foo(_Bool); static _Bool (*foop)(_Bool);")
     lib = ffi.verify("""
         struct foo_s { _Bool x; };
         int foo(int arg) {
@@ -1462,7 +1462,7 @@
         _Bool _foofunc(_Bool x) {
             return !x;
         }
-        _Bool (*foop)(_Bool) = _foofunc;
+        static _Bool (*foop)(_Bool) = _foofunc;
     """)
     p = ffi.new("struct foo_s *")
     p.x = 1
@@ -1653,7 +1653,7 @@
 
 def test_FILE_stored_explicitly():
     ffi = FFI()
-    ffi.cdef("int myprintf11(const char *, int); FILE *myfile;")
+    ffi.cdef("int myprintf11(const char *, int); extern FILE *myfile;")
     lib = ffi.verify("""
         #include <stdio.h>
         FILE *myfile;
@@ -1679,19 +1679,19 @@
 
 def test_global_array_with_missing_length():
     ffi = FFI()
-    ffi.cdef("int fooarray[];")
+    ffi.cdef("extern int fooarray[];")
     lib = ffi.verify("int fooarray[50];")
     assert repr(lib.fooarray).startswith("<cdata 'int *'")
 
 def test_global_array_with_dotdotdot_length():
     ffi = FFI()
-    ffi.cdef("int fooarray[...];")
+    ffi.cdef("extern int fooarray[...];")
     lib = ffi.verify("int fooarray[50];")
     assert repr(lib.fooarray).startswith("<cdata 'int[50]'")
 
 def test_bad_global_array_with_dotdotdot_length():
     ffi = FFI()
-    ffi.cdef("int fooarray[...];")
+    ffi.cdef("extern int fooarray[...];")
     py.test.raises(VerificationError, ffi.verify, "char fooarray[23];")
 
 def test_struct_containing_struct():
@@ -1812,7 +1812,7 @@
 def test_callback_indirection():
     ffi = FFI()
     ffi.cdef("""
-        int (*python_callback)(int how_many, int *values);
+        static int (*python_callback)(int how_many, int *values);
         int (*const c_callback)(int,...);   /* pass this ptr to C routines */
         int some_c_function(int(*cb)(int,...));
     """)
@@ -1946,24 +1946,24 @@
 
 def test_bug_const_char_ptr_array_1():
     ffi = FFI()
-    ffi.cdef("""const char *a[...];""")
+    ffi.cdef("""extern const char *a[...];""")
     lib = ffi.verify("""const char *a[5];""")
     assert repr(ffi.typeof(lib.a)) == "<ctype 'char *[5]'>"
 
 def test_bug_const_char_ptr_array_2():
     from cffi import FFI     # ignore warnings
     ffi = FFI()
-    ffi.cdef("""const int a[];""")
+    ffi.cdef("""extern const int a[];""")
     lib = ffi.verify("""const int a[5];""")
     assert repr(ffi.typeof(lib.a)) == "<ctype 'int *'>"
 
 def _test_various_calls(force_libffi):
     cdef_source = """
-    int xvalue;
-    long long ivalue, rvalue;
-    float fvalue;
-    double dvalue;
-    long double Dvalue;
+    extern int xvalue;
+    extern long long ivalue, rvalue;
+    extern float fvalue;
+    extern double dvalue;
+    extern long double Dvalue;
     signed char tf_bb(signed char x, signed char c);
     unsigned char tf_bB(signed char x, unsigned char c);
     short tf_bh(signed char x, short c);
@@ -2147,7 +2147,7 @@
     # exported symbols as well.  So we must not export a simple name
     # like 'foo'!
     ffi1 = FFI()
-    ffi1.cdef("int foo_verify_dlopen_flags;")
+    ffi1.cdef("extern int foo_verify_dlopen_flags;")
 
     lib1 = ffi1.verify("int foo_verify_dlopen_flags;",
                        flags=ffi1.RTLD_GLOBAL | ffi1.RTLD_LAZY)
@@ -2161,7 +2161,7 @@
 def get_second_lib():
     # Hack, using modulename makes the test fail
     ffi2 = FFI()
-    ffi2.cdef("int foo_verify_dlopen_flags;")
+    ffi2.cdef("extern int foo_verify_dlopen_flags;")
     lib2 = ffi2.verify("int foo_verify_dlopen_flags;",
                        flags=ffi2.RTLD_GLOBAL | ffi2.RTLD_LAZY)
     return lib2
diff --git a/testing/cffi1/test_dlopen.py b/testing/cffi1/test_dlopen.py
--- a/testing/cffi1/test_dlopen.py
+++ b/testing/cffi1/test_dlopen.py
@@ -6,7 +6,7 @@
 
 def test_simple():
     ffi = FFI()
-    ffi.cdef("int close(int); static const int BB = 42; int somevar;")
+    ffi.cdef("int close(int); static const int BB = 42; extern int somevar;")
     target = udir.join('test_simple.py')
     make_py_source(ffi, 'test_simple', str(target))
     assert target.read() == r"""# auto-generated file
@@ -196,7 +196,7 @@
 
 def test_global_var():
     ffi = FFI()
-    ffi.cdef("int myglob;")
+    ffi.cdef("extern int myglob;")
     target = udir.join('test_global_var.py')
     make_py_source(ffi, 'test_global_var', str(target))
     assert target.read() == r"""# auto-generated file
diff --git a/testing/cffi1/test_new_ffi_1.py b/testing/cffi1/test_new_ffi_1.py
--- a/testing/cffi1/test_new_ffi_1.py
+++ b/testing/cffi1/test_new_ffi_1.py
@@ -1779,7 +1779,7 @@
 
     def test_import_from_lib(self):
         ffi2 = cffi.FFI()
-        ffi2.cdef("int myfunc(int); int myvar;\n#define MYFOO ...\n")
+        ffi2.cdef("int myfunc(int); extern int myvar;\n#define MYFOO ...\n")
         outputfilename = recompile(ffi2, "_test_import_from_lib",
                                    "int myfunc(int x) { return x + 1; }\n"
                                    "int myvar = -5;\n"
diff --git a/testing/cffi1/test_re_python.py b/testing/cffi1/test_re_python.py
--- a/testing/cffi1/test_re_python.py
+++ b/testing/cffi1/test_re_python.py
@@ -63,11 +63,11 @@
     #define BIGNEG -420000000000L
     int add42(int);
     int add43(int, ...);
-    int globalvar42;
+    extern int globalvar42;
     const int globalconst42;
     const char *const globalconsthello;
     int no_such_function(int);
-    int no_such_globalvar;
+    extern int no_such_globalvar;
     struct foo_s;
     typedef struct bar_s { int x; signed char a[]; } bar_t;
     enum foo_e { AA, BB, CC };
diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -83,7 +83,7 @@
                      "(FUNCTION 1)(PRIMITIVE 7)(FUNCTION_END 1)(POINTER 0)")
 
 def test_type_table_array():
-    check_type_table("int a[100];",
+    check_type_table("extern int a[100];",
                      "(PRIMITIVE 7)(ARRAY 0)(None 100)")
 
 def test_type_table_typedef():
@@ -158,7 +158,7 @@
 
 def test_global_var_array():
     ffi = FFI()
-    ffi.cdef("int a[100];")
+    ffi.cdef("extern int a[100];")
     lib = verify(ffi, 'test_global_var_array', 'int a[100] = { 9999 };')
     lib.a[42] = 123456
     assert lib.a[42] == 123456
@@ -182,7 +182,7 @@
 
 def test_global_var_int():
     ffi = FFI()
-    ffi.cdef("int a, b, c;")
+    ffi.cdef("extern int a, b, c;")
     lib = verify(ffi, 'test_global_var_int', 'int a = 999, b, c;')
     assert lib.a == 999
     lib.a -= 1001
@@ -283,7 +283,7 @@
 
 def test_dir():
     ffi = FFI()
-    ffi.cdef("int ff(int); int aa; static const int my_constant;")
+    ffi.cdef("int ff(int); extern int aa; static const int my_constant;")
     lib = verify(ffi, 'test_dir', """
         #define my_constant  (-45)
         int aa;
@@ -405,7 +405,7 @@
 
 def test_dotdotdot_global_array():
     ffi = FFI()
-    ffi.cdef("int aa[...]; int bb[...];")
+    ffi.cdef("extern int aa[...]; extern int bb[...];")
     lib = verify(ffi, 'test_dotdotdot_global_array',
                  "int aa[41]; int bb[12];")
     assert ffi.sizeof(lib.aa) == 41 * 4
@@ -560,37 +560,37 @@
 
 def test_bad_size_of_global_1():
     ffi = FFI()
-    ffi.cdef("short glob;")
+    ffi.cdef("extern short glob;")
     py.test.raises(VerificationError, verify, ffi,
                    "test_bad_size_of_global_1", "long glob;")
 
 def test_bad_size_of_global_2():
     ffi = FFI()
-    ffi.cdef("int glob[10];")
+    ffi.cdef("extern int glob[10];")
     py.test.raises(VerificationError, verify, ffi,
                    "test_bad_size_of_global_2", "int glob[9];")
 
 def test_unspecified_size_of_global_1():
     ffi = FFI()
-    ffi.cdef("int glob[];")
+    ffi.cdef("extern int glob[];")
     lib = verify(ffi, "test_unspecified_size_of_global_1", "int glob[10];")
     assert ffi.typeof(lib.glob) == ffi.typeof("int *")
 
 def test_unspecified_size_of_global_2():
     ffi = FFI()
-    ffi.cdef("int glob[][5];")
+    ffi.cdef("extern int glob[][5];")
     lib = verify(ffi, "test_unspecified_size_of_global_2", "int glob[10][5];")
     assert ffi.typeof(lib.glob) == ffi.typeof("int(*)[5]")
 
 def test_unspecified_size_of_global_3():
     ffi = FFI()
-    ffi.cdef("int glob[][...];")
+    ffi.cdef("extern int glob[][...];")
     lib = verify(ffi, "test_unspecified_size_of_global_3", "int glob[10][5];")
     assert ffi.typeof(lib.glob) == ffi.typeof("int(*)[5]")
 
 def test_unspecified_size_of_global_4():
     ffi = FFI()
-    ffi.cdef("int glob[...][...];")
+    ffi.cdef("extern int glob[...][...];")
     lib = verify(ffi, "test_unspecified_size_of_global_4", "int glob[10][5];")
     assert ffi.typeof(lib.glob) == ffi.typeof("int[10][5]")
 
@@ -813,7 +813,7 @@
 def test_address_of_global_var():
     ffi = FFI()
     ffi.cdef("""
-        long bottom, bottoms[2];
+        extern long bottom, bottoms[2];
         long FetchRectBottom(void);
         long FetchRectBottoms1(void);
         #define FOOBAR 42
@@ -968,7 +968,7 @@
     ffi = FFI()
     ffi.cdef("""
         typedef ... opaque_t;
-        opaque_t globvar;
+        extern opaque_t globvar;
     """)
     lib = verify(ffi, 'test_variable_of_unknown_size', """
         typedef char opaque_t[6];
@@ -1013,7 +1013,7 @@
 def test_call_with_incomplete_structs():
     ffi = FFI()
     ffi.cdef("typedef struct {...;} foo_t; "
-             "foo_t myglob; "
+             "extern foo_t myglob; "
              "foo_t increment(foo_t s); "
              "double getx(foo_t s);")
     lib = verify(ffi, 'test_call_with_incomplete_structs', """
@@ -1057,7 +1057,7 @@
 
 def test_global_var_array_2():
     ffi = FFI()
-    ffi.cdef("int a[...][...];")
+    ffi.cdef("extern int a[...][...];")
     lib = verify(ffi, 'test_global_var_array_2', 'int a[10][8];')
     lib.a[9][7] = 123456
     assert lib.a[9][7] == 123456
@@ -1070,7 +1070,7 @@
 
 def test_global_var_array_3():
     ffi = FFI()
-    ffi.cdef("int a[][...];")
+    ffi.cdef("extern int a[][...];")
     lib = verify(ffi, 'test_global_var_array_3', 'int a[10][8];')
     lib.a[9][7] = 123456
     assert lib.a[9][7] == 123456
@@ -1081,7 +1081,7 @@
 
 def test_global_var_array_4():
     ffi = FFI()
-    ffi.cdef("int a[10][...];")
+    ffi.cdef("extern int a[10][...];")
     lib = verify(ffi, 'test_global_var_array_4', 'int a[10][8];')
     lib.a[9][7] = 123456
     assert lib.a[9][7] == 123456
@@ -1204,7 +1204,7 @@
 
 def test_import_from_lib():
     ffi = FFI()
-    ffi.cdef("int mybar(int); int myvar;\n#define MYFOO ...")
+    ffi.cdef("int mybar(int); static int myvar;\n#define MYFOO ...")
     lib = verify(ffi, 'test_import_from_lib',
                  "#define MYFOO 42\n"
                  "static int mybar(int x) { return x + 1; }\n"
@@ -1220,7 +1220,7 @@
 
 def test_macro_var_callback():
     ffi = FFI()
-    ffi.cdef("int my_value; int *(*get_my_value)(void);")
+    ffi.cdef("extern int my_value; extern int *(*get_my_value)(void);")
     lib = verify(ffi, 'test_macro_var_callback',
                  "int *(*get_my_value)(void);\n"
                  "#define my_value (*get_my_value())")
@@ -1335,7 +1335,7 @@
 
 def test_const_function_type_args():
     ffi = FFI()
-    ffi.cdef("""int (*foobar)(const int a, const int *b, const int c[]);""")
+    ffi.cdef("""extern int(*foobar)(const int a,const int*b,const int c[]);""")
     lib = verify(ffi, 'test_const_function_type_args', """
         int (*foobar)(const int a, const int *b, const int c[]);
     """)
@@ -1625,7 +1625,7 @@
 
 def test_extern_python_bogus_name():
     ffi = FFI()
-    ffi.cdef("int abc;")
+    ffi.cdef("extern int abc;")
     lib = verify(ffi, 'test_extern_python_bogus_name', "int abc;")
     def fn():
         pass
@@ -1786,8 +1786,8 @@
     ffi.cdef("""
         extern "Python" int __stdcall foo(int);
         extern "Python" int WINAPI bar(int);
-        int (__stdcall * mycb1)(int);
-        int indirect_call(int);
+        static int (__stdcall * mycb1)(int);
+        static int indirect_call(int);
     """)
     lib = verify(ffi, 'test_extern_python_stdcall', """
         #ifndef _MSC_VER
@@ -1855,7 +1855,7 @@
 
 def test_introspect_global_var():
     ffi = FFI()
-    ffi.cdef("float g1;")
+    ffi.cdef("extern float g1;")
     lib = verify(ffi, 'test_introspect_global_var', """
         float g1;
     """)
@@ -1866,7 +1866,7 @@
 
 def test_introspect_global_var_array():
     ffi = FFI()
-    ffi.cdef("float g1[100];")
+    ffi.cdef("extern float g1[100];")
     lib = verify(ffi, 'test_introspect_global_var_array', """
         float g1[100];
     """)
@@ -2089,7 +2089,7 @@
     ffi = FFI()
     ffi.cdef("""
         typedef int foo_t[...], bar_t[...];
-        int gv[...];
+        extern int gv[...];
         typedef int mat_t[...][...];
         typedef int vmat_t[][...];
         """)
diff --git a/testing/cffi1/test_verify1.py b/testing/cffi1/test_verify1.py
--- a/testing/cffi1/test_verify1.py
+++ b/testing/cffi1/test_verify1.py
@@ -267,7 +267,7 @@
 def test_var_signed_integer_types():
     ffi = FFI()
     lst = all_signed_integer_types(ffi)
-    csource = "\n".join(["%s somevar_%s;" % (tp, tp.replace(' ', '_'))
+    csource = "\n".join(["static %s somevar_%s;" % (tp, tp.replace(' ', '_'))
                          for tp in lst])
     ffi.cdef(csource)
     lib = ffi.verify(csource)
@@ -286,7 +286,7 @@
 def test_var_unsigned_integer_types():
     ffi = FFI()
     lst = all_unsigned_integer_types(ffi)
-    csource = "\n".join(["%s somevar_%s;" % (tp, tp.replace(' ', '_'))
+    csource = "\n".join(["static %s somevar_%s;" % (tp, tp.replace(' ', '_'))
                          for tp in lst])
     ffi.cdef(csource)
     lib = ffi.verify(csource)
@@ -790,8 +790,8 @@
 
 def test_access_variable():
     ffi = FFI()
-    ffi.cdef("int foo(void);\n"
-             "int somenumber;")
+    ffi.cdef("static int foo(void);\n"
+             "static int somenumber;")
     lib = ffi.verify("""
         static int somenumber = 2;
         static int foo(void) {
@@ -808,7 +808,7 @@
 def test_access_address_of_variable():
     # access the address of 'somenumber': need a trick
     ffi = FFI()
-    ffi.cdef("int somenumber; static int *const somenumberptr;")
+    ffi.cdef("static int somenumber; static int *const somenumberptr;")
     lib = ffi.verify("""
         static int somenumber = 2;
         #define somenumberptr (&somenumber)
@@ -820,8 +820,8 @@
 
 def test_access_array_variable(length=5):
     ffi = FFI()
-    ffi.cdef("int foo(int);\n"
-             "int somenumber[%s];" % (length,))
+    ffi.cdef("static int foo(int);\n"
+             "static int somenumber[%s];" % (length,))
     lib = ffi.verify("""
         static int somenumber[] = {2, 2, 3, 4, 5};
         static int foo(int i) {
@@ -852,8 +852,8 @@
 def test_access_struct_variable():
     ffi = FFI()
     ffi.cdef("struct foo { int x; ...; };\n"
-             "int foo(int);\n"
-             "struct foo stuff;")
+             "static int foo(int);\n"
+             "static struct foo stuff;")
     lib = ffi.verify("""
         struct foo { int x, y, z; };
         static struct foo stuff = {2, 5, 8};
@@ -877,9 +877,9 @@
 
 def test_access_callback():
     ffi = FFI()
-    ffi.cdef("int (*cb)(int);\n"
-             "int foo(int);\n"
-             "void reset_cb(void);")
+    ffi.cdef("static int (*cb)(int);\n"
+             "static int foo(int);\n"
+             "static void reset_cb(void);")
     lib = ffi.verify("""
         static int g(int x) { return x * 7; }
         static int (*cb)(int);
@@ -895,9 +895,9 @@
 def test_access_callback_function_typedef():
     ffi = FFI()
     ffi.cdef("typedef int mycallback_t(int);\n"
-             "mycallback_t *cb;\n"
-             "int foo(int);\n"
-             "void reset_cb(void);")
+             "static mycallback_t *cb;\n"
+             "static int foo(int);\n"
+             "static void reset_cb(void);")
     lib = ffi.verify("""
         static int g(int x) { return x * 7; }
         static int (*cb)(int);
@@ -1038,7 +1038,7 @@
 def test_autofilled_struct_as_argument_dynamic():
     ffi = FFI()
     ffi.cdef("struct foo_s { long a; ...; };\n"
-             "int (*foo)(struct foo_s);")
+             "static int (*foo)(struct foo_s);")
     lib = ffi.verify("""
         struct foo_s {
             double b;
@@ -1047,7 +1047,7 @@
         int foo1(struct foo_s s) {
             return (int)s.a - (int)s.b;
         }
-        int (*foo)(struct foo_s s) = &foo1;
+        static int (*foo)(struct foo_s s) = &foo1;
     """)
     e = py.test.raises(NotImplementedError, lib.foo, "?")
     msg = ("ctype 'struct foo_s' not supported as argument.  It is a struct "
@@ -1423,7 +1423,7 @@
         py.test.skip("_Bool not in MSVC")
     ffi = FFI()
     ffi.cdef("struct foo_s { _Bool x; };"
-             "_Bool foo(_Bool); _Bool (*foop)(_Bool);")
+             "_Bool foo(_Bool); static _Bool (*foop)(_Bool);")
     lib = ffi.verify("""
         struct foo_s { _Bool x; };
         int foo(int arg) {
@@ -1432,7 +1432,7 @@
         _Bool _foofunc(_Bool x) {
             return !x;
         }
-        _Bool (*foop)(_Bool) = _foofunc;
+        static _Bool (*foop)(_Bool) = _foofunc;
     """)
     p = ffi.new("struct foo_s *")
     p.x = 1
@@ -1617,7 +1617,7 @@
 
 def test_FILE_stored_explicitly():
     ffi = FFI()
-    ffi.cdef("int myprintf11(const char *, int); FILE *myfile;")
+    ffi.cdef("int myprintf11(const char *, int); extern FILE *myfile;")
     lib = ffi.verify("""
         #include <stdio.h>
         FILE *myfile;
@@ -1643,13 +1643,13 @@
 
 def test_global_array_with_missing_length():
     ffi = FFI()
-    ffi.cdef("int fooarray[];")
+    ffi.cdef("extern int fooarray[];")
     lib = ffi.verify("int fooarray[50];")
     assert repr(lib.fooarray).startswith("<cdata 'int *'")
 
 def test_global_array_with_dotdotdot_length():
     ffi = FFI()
-    ffi.cdef("int fooarray[...];")
+    ffi.cdef("extern int fooarray[...];")
     lib = ffi.verify("int fooarray[50];")
     assert repr(lib.fooarray).startswith("<cdata 'int[50]'")
 
@@ -1657,7 +1657,7 @@
     py.test.xfail("was detected only because 23 bytes cannot be divided by 4; "
                   "redo more generally")
     ffi = FFI()
-    ffi.cdef("int fooarray[...];")
+    ffi.cdef("extern int fooarray[...];")
     py.test.raises(VerificationError, ffi.verify, "char fooarray[23];")
 
 def test_struct_containing_struct():
@@ -1778,7 +1778,7 @@
 def test_callback_indirection():
     ffi = FFI()
     ffi.cdef("""
-        int (*python_callback)(int how_many, int *values);
+        static int (*python_callback)(int how_many, int *values);
         int (*const c_callback)(int,...);   /* pass this ptr to C routines */
         int some_c_function(int(*cb)(int,...));
     """)
@@ -1912,23 +1912,23 @@
 
 def test_bug_const_char_ptr_array_1():
     ffi = FFI()
-    ffi.cdef("""const char *a[...];""")
+    ffi.cdef("""extern const char *a[...];""")
     lib = ffi.verify("""const char *a[5];""")
     assert repr(ffi.typeof(lib.a)) == "<ctype 'char *[5]'>"
 
 def test_bug_const_char_ptr_array_2():
     ffi = FFI()
-    ffi.cdef("""const int a[];""")
+    ffi.cdef("""extern const int a[];""")
     lib = ffi.verify("""const int a[5];""")
     assert repr(ffi.typeof(lib.a)) == "<ctype 'int *'>"
 
 def _test_various_calls(force_libffi):
     cdef_source = """
-    int xvalue;
-    long long ivalue, rvalue;
-    float fvalue;
-    double dvalue;
-    long double Dvalue;
+    extern int xvalue;
+    extern long long ivalue, rvalue;
+    extern float fvalue;
+    extern double dvalue;
+    extern long double Dvalue;
     signed char tf_bb(signed char x, signed char c);
     unsigned char tf_bB(signed char x, unsigned char c);
     short tf_bh(signed char x, short c);
@@ -2111,7 +2111,7 @@
     old = sys.getdlopenflags()
     try:
         ffi1 = FFI()
-        ffi1.cdef("int foo_verify_dlopen_flags_1;")
+        ffi1.cdef("extern int foo_verify_dlopen_flags_1;")
         sys.setdlopenflags(ffi1.RTLD_GLOBAL | ffi1.RTLD_NOW)
         lib1 = ffi1.verify("int foo_verify_dlopen_flags_1;")
     finally:
@@ -2252,7 +2252,7 @@
 
 def test_macro_var():
     ffi = FFI()
-    ffi.cdef("int myarray[50], my_value;")
+    ffi.cdef("extern int myarray[50], my_value;")
     lib = ffi.verify("""
         int myarray[50];
         int *get_my_value(void) {
diff --git a/testing/embedding/add_recursive.py b/testing/embedding/add_recursive.py
--- a/testing/embedding/add_recursive.py
+++ b/testing/embedding/add_recursive.py
@@ -3,7 +3,7 @@
 ffi = cffi.FFI()
 
 ffi.embedding_api("""
-    int (*my_callback)(int);
+    extern int (*my_callback)(int);
     int add_rec(int, int);
 """)
 

From pypy.commits at gmail.com  Mon Oct 14 05:07:37 2019
From: pypy.commits at gmail.com (arigo)
Date: Mon, 14 Oct 2019 02:07:37 -0700 (PDT)
Subject: [pypy-commit] pypy fix-descrmismatch-crash: frozenset.__init__()
 does nothing on CPython, and can be called with random
Message-ID: <5da43ad9.1c69fb81.1d381.fa1b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: fix-descrmismatch-crash
Changeset: r97769:8c25c3e5ab7d
Date: 2019-10-14 11:06 +0200
http://bitbucket.org/pypy/pypy/changeset/8c25c3e5ab7d/

Log:	frozenset.__init__() does nothing on CPython, and can be called with
	random arguments.

diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -1014,7 +1014,7 @@
 
     def test_fast_path_crash(self):
         # issue bb-3091 crash in BuiltinCodePassThroughArguments0.funcrun
-        for obj in (dict, set, frozenset):
+        for obj in (dict, set):
             with raises(TypeError) as excinfo:
                 if self.runappdirect:
                     msg_fmt = "'%s' object but received a '%s'"
diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -578,11 +578,6 @@
 class W_FrozensetObject(W_BaseSetObject):
     hash = 0
 
-    #overridden here so the error is reported correctly
-    def __init__(self, space, w_iterable=None):
-        """Initialize the frozenset by taking ownership of 'setdata'."""
-        W_BaseSetObject.__init__(self, space, w_iterable)
-
     def _cleanup_(self):
         # in case there are frozenset objects existing during
         # translation, make sure we don't translate a cached hash
@@ -650,7 +645,6 @@
 
 Build an immutable unordered collection.""",
     __new__ = gateway.interp2app(W_FrozensetObject.descr_new2),
-    __init__ = gateway.interp2app(W_FrozensetObject.descr_init),
     __repr__ = gateway.interp2app(W_BaseSetObject.descr_repr),
     __hash__ = gateway.interp2app(W_FrozensetObject.descr_hash),
     __cmp__ = gateway.interp2app(W_BaseSetObject.descr_cmp),
diff --git a/pypy/objspace/std/test/test_setobject.py b/pypy/objspace/std/test/test_setobject.py
--- a/pypy/objspace/std/test/test_setobject.py
+++ b/pypy/objspace/std/test/test_setobject.py
@@ -1037,3 +1037,8 @@
            raise ValueError
            yield 1
         raises(ValueError, set, f())
+
+    def test_frozenset_init_does_nothing(self):
+        f = frozenset([1, 2, 3])
+        f.__init__(4, 5, 6)
+        assert f == frozenset([1, 2, 3])

From pypy.commits at gmail.com  Mon Oct 14 05:25:56 2019
From: pypy.commits at gmail.com (arigo)
Date: Mon, 14 Oct 2019 02:25:56 -0700 (PDT)
Subject: [pypy-commit] pypy fix-descrmismatch-crash: Make a new helper
 method for the common code, and move there the logic that
Message-ID: <5da43f24.1c69fb81.8c71.851a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: fix-descrmismatch-crash
Changeset: r97770:9acddd120236
Date: 2019-10-14 11:25 +0200
http://bitbucket.org/pypy/pypy/changeset/9acddd120236/

Log:	Make a new helper method for the common code, and move there the
	logic that should safely fetch the first argument. Includes tests
	but like the old SystemError these new cases are probably not
	reachable from app-level

diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -755,10 +755,7 @@
         except DescrMismatch:
             if w_obj is not None:
                 args = args.prepend(w_obj)
-            return scope_w[0].descr_call_mismatch(space,
-                                                  self.descrmismatch_op,
-                                                  self.descr_reqcls,
-                                                  args)
+            return self.descr_call_mismatch(space, args)
         except Exception as e:
             self.handle_exception(space, e)
             w_result = None
@@ -766,6 +763,15 @@
             w_result = space.w_None
         return w_result
 
+    def descr_call_mismatch(self, space, args):
+        w_obj = args.firstarg()
+        if w_obj is None:
+            raise oefmt(space.w_SystemError, "unexpected DescrMismatch error")
+        return w_obj.descr_call_mismatch(space,
+                                         self.descrmismatch_op,
+                                         self.descr_reqcls,
+                                         args)
+
     def handle_exception(self, space, e):
         try:
             if not we_are_translated():
@@ -788,10 +794,7 @@
         try:
             w_result = self.func__args__(space, args)
         except DescrMismatch:
-            return args.firstarg().descr_call_mismatch(space,
-                                                  self.descrmismatch_op,
-                                                  self.descr_reqcls,
-                                                  args)
+            return self.descr_call_mismatch(space, args)
         except Exception as e:
             self.handle_exception(space, e)
             w_result = None
@@ -809,10 +812,7 @@
         try:
             w_result = self.func__args__(space, w_obj, args)
         except DescrMismatch:
-            return w_obj.descr_call_mismatch(space,
-                                                  self.descrmismatch_op,
-                                                  self.descr_reqcls,
-                                                  args.prepend(w_obj))
+            return self.descr_call_mismatch(space, args.prepend(w_obj))
         except Exception as e:
             self.handle_exception(space, e)
             w_result = None
@@ -852,9 +852,7 @@
         try:
             w_result = self.fastfunc_1(space, w1)
         except DescrMismatch:
-            return w1.descr_call_mismatch(space,
-                                          self.descrmismatch_op,
-                                          self.descr_reqcls,
+            return self.descr_call_mismatch(space,
                                           Arguments(space, [w1]))
         except Exception as e:
             self.handle_exception(space, e)
@@ -878,9 +876,7 @@
         try:
             w_result = self.fastfunc_2(space, w1, w2)
         except DescrMismatch:
-            return w1.descr_call_mismatch(space,
-                                          self.descrmismatch_op,
-                                          self.descr_reqcls,
+            return self.descr_call_mismatch(space,
                                           Arguments(space, [w1, w2]))
         except Exception as e:
             self.handle_exception(space, e)
@@ -905,9 +901,7 @@
         try:
             w_result = self.fastfunc_3(space, w1, w2, w3)
         except DescrMismatch:
-            return w1.descr_call_mismatch(space,
-                                          self.descrmismatch_op,
-                                          self.descr_reqcls,
+            return self.descr_call_mismatch(space,
                                           Arguments(space, [w1, w2, w3]))
         except Exception as e:
             self.handle_exception(space, e)
@@ -933,9 +927,7 @@
         try:
             w_result = self.fastfunc_4(space, w1, w2, w3, w4)
         except DescrMismatch:
-            return w1.descr_call_mismatch(space,
-                                          self.descrmismatch_op,
-                                          self.descr_reqcls,
+            return self.descr_call_mismatch(space,
                                           Arguments(space,
                                                     [w1, w2, w3, w4]))
         except Exception as e:
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -966,6 +966,29 @@
         # white-box check for opt
         assert called[0] is args
 
+    def test_base_regular_descr_mismatch(self):
+        space = self.space
+
+        def f():
+            raise gateway.DescrMismatch
+
+        w_f = space.wrap(gateway.interp2app_temp(f,
+                         unwrap_spec=[]))
+        args = argument.Arguments(space, [])
+        space.raises_w(space.w_SystemError, space.call_args, w_f, args)
+
+    def test_pass_trough_arguments0_descr_mismatch(self):
+        space = self.space
+
+        def f(space, __args__):
+            raise gateway.DescrMismatch
+
+        w_f = space.wrap(gateway.interp2app_temp(f,
+                         unwrap_spec=[gateway.ObjSpace,
+                                      gateway.Arguments]))
+        args = argument.Arguments(space, [])
+        space.raises_w(space.w_SystemError, space.call_args, w_f, args)
+
 
 class AppTestKeywordsToBuiltinSanity(object):
     def test_type(self):

From pypy.commits at gmail.com  Mon Oct 14 05:41:03 2019
From: pypy.commits at gmail.com (arigo)
Date: Mon, 14 Oct 2019 02:41:03 -0700 (PDT)
Subject: [pypy-commit] pypy fix-descrmismatch-crash: Improve the error
 message by using the class name 'set-or-frozenset'
Message-ID: <5da442af.1c69fb81.4bdfc.3d9a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: fix-descrmismatch-crash
Changeset: r97771:78855fca6aa9
Date: 2019-10-14 11:40 +0200
http://bitbucket.org/pypy/pypy/changeset/78855fca6aa9/

Log:	Improve the error message by using the class name 'set-or-frozenset'

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -419,6 +419,8 @@
 
 @specialize.memo()
 def wrappable_class_name(Class):
+    if 'exact_class_applevel_name' in Class.__dict__:
+        return Class.exact_class_applevel_name
     try:
         return Class.typedef.name
     except AttributeError:
diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -20,6 +20,7 @@
 
 class W_BaseSetObject(W_Root):
     typedef = None
+    exact_class_applevel_name = 'set-or-frozenset'
 
     def __init__(self, space, w_iterable=None):
         """Initialize the set by taking ownership of 'setdata'."""
diff --git a/pypy/objspace/std/test/test_setobject.py b/pypy/objspace/std/test/test_setobject.py
--- a/pypy/objspace/std/test/test_setobject.py
+++ b/pypy/objspace/std/test/test_setobject.py
@@ -1042,3 +1042,10 @@
         f = frozenset([1, 2, 3])
         f.__init__(4, 5, 6)
         assert f == frozenset([1, 2, 3])
+
+    def test_error_message_wrong_self(self):
+        e = raises(TypeError, frozenset.copy, 42)
+        assert "frozenset" in str(e.value)
+        if hasattr(frozenset.copy, 'im_func'):
+            e = raises(TypeError, frozenset.copy.im_func, 42)
+            assert "'set-or-frozenset'" in str(e.value)

From pypy.commits at gmail.com  Mon Oct 14 05:51:32 2019
From: pypy.commits at gmail.com (arigo)
Date: Mon, 14 Oct 2019 02:51:32 -0700 (PDT)
Subject: [pypy-commit] pypy fix-descrmismatch-crash: fix the error message
 also for W_AbstractBytesObject
Message-ID: <5da44524.1c69fb81.23669.fdd8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: fix-descrmismatch-crash
Changeset: r97772:c13fae180ec6
Date: 2019-10-14 11:45 +0200
http://bitbucket.org/pypy/pypy/changeset/c13fae180ec6/

Log:	fix the error message also for W_AbstractBytesObject

diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -24,6 +24,7 @@
 
 class W_AbstractBytesObject(W_Root):
     __slots__ = ()
+    exact_class_applevel_name = 'str'
 
     def is_w(self, space, w_other):
         if not isinstance(w_other, W_AbstractBytesObject):
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -933,3 +933,10 @@
     def test_add(self):
         assert 'abc' + 'abc' == 'abcabc'
         assert isinstance('abc' + u'\u03a3', unicode)
+
+    def test_error_message_wrong_self(self):
+        e = raises(TypeError, bytes.upper, 42)
+        assert "str" in str(e.value)
+        if hasattr(bytes.upper, 'im_func'):
+            e = raises(TypeError, bytes.upper.im_func, 42)
+            assert "'str'" in str(e.value)

From pypy.commits at gmail.com  Mon Oct 14 05:51:34 2019
From: pypy.commits at gmail.com (arigo)
Date: Mon, 14 Oct 2019 02:51:34 -0700 (PDT)
Subject: [pypy-commit] pypy fix-descrmismatch-crash: same fix for
 weakref-or-proxy
Message-ID: <5da44526.1c69fb81.4a165.22c9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: fix-descrmismatch-crash
Changeset: r97773:cc73504e9f94
Date: 2019-10-14 11:48 +0200
http://bitbucket.org/pypy/pypy/changeset/cc73504e9f94/

Log:	same fix for weakref-or-proxy

diff --git a/pypy/module/_weakref/interp__weakref.py b/pypy/module/_weakref/interp__weakref.py
--- a/pypy/module/_weakref/interp__weakref.py
+++ b/pypy/module/_weakref/interp__weakref.py
@@ -156,6 +156,8 @@
 
 
 class W_WeakrefBase(W_Root):
+    exact_class_applevel_name = 'weakref-or-proxy'
+
     def __init__(self, space, w_obj, w_callable):
         assert w_callable is not space.w_None    # should be really None
         self.space = space
diff --git a/pypy/module/_weakref/test/test_weakref.py b/pypy/module/_weakref/test/test_weakref.py
--- a/pypy/module/_weakref/test/test_weakref.py
+++ b/pypy/module/_weakref/test/test_weakref.py
@@ -543,3 +543,12 @@
         p1[42] = p2
         assert a1.setkey == 42
         assert a1.setvalue is p2
+
+    def test_error_message_wrong_self(self):
+        import _weakref
+        unboundmeth = _weakref.ref.__repr__
+        e = raises(TypeError, unboundmeth, 42)
+        assert "weakref" in str(e.value)
+        if hasattr(unboundmeth, 'im_func'):
+            e = raises(TypeError, unboundmeth.im_func, 42)
+            assert "'weakref-or-proxy'" in str(e.value)

From pypy.commits at gmail.com  Mon Oct 14 05:51:36 2019
From: pypy.commits at gmail.com (arigo)
Date: Mon, 14 Oct 2019 02:51:36 -0700 (PDT)
Subject: [pypy-commit] pypy fix-descrmismatch-crash: Same fix for cStringIO
Message-ID: <5da44528.1c69fb81.a67ba.cf65@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: fix-descrmismatch-crash
Changeset: r97774:56cde2db6b50
Date: 2019-10-14 11:50 +0200
http://bitbucket.org/pypy/pypy/changeset/56cde2db6b50/

Log:	Same fix for cStringIO

diff --git a/pypy/module/cStringIO/interp_stringio.py b/pypy/module/cStringIO/interp_stringio.py
--- a/pypy/module/cStringIO/interp_stringio.py
+++ b/pypy/module/cStringIO/interp_stringio.py
@@ -7,6 +7,8 @@
 
 
 class W_InputOutputType(W_Root):
+    exact_class_applevel_name = "StringI-or-StringO"
+
     softspace = 0    # part of the file object API
 
     def descr___iter__(self):
diff --git a/pypy/module/cStringIO/test/test_interp_stringio.py b/pypy/module/cStringIO/test/test_interp_stringio.py
--- a/pypy/module/cStringIO/test/test_interp_stringio.py
+++ b/pypy/module/cStringIO/test/test_interp_stringio.py
@@ -204,3 +204,12 @@
         import cStringIO
         assert type(cStringIO.StringIO()) is cStringIO.OutputType
         assert type(cStringIO.StringIO('')) is cStringIO.InputType
+
+    def test_error_message_wrong_self(self):
+        import cStringIO
+        unboundmeth = cStringIO.InputType.close
+        e = raises(TypeError, unboundmeth, 42)
+        assert "StringI" in str(e.value)
+        if hasattr(unboundmeth, 'im_func'):
+            e = raises(TypeError, unboundmeth.im_func, 42)
+            assert "'StringI-or-StringO'" in str(e.value)

From pypy.commits at gmail.com  Mon Oct 14 06:00:42 2019
From: pypy.commits at gmail.com (arigo)
Date: Mon, 14 Oct 2019 03:00:42 -0700 (PDT)
Subject: [pypy-commit] pypy fix-descrmismatch-crash: Some more cases of
 W_AbstractXxx but where the test already passes
Message-ID: <5da4474a.1c69fb81.6c584.4bd1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: fix-descrmismatch-crash
Changeset: r97775:c2572a2d1270
Date: 2019-10-14 11:59 +0200
http://bitbucket.org/pypy/pypy/changeset/c2572a2d1270/

Log:	Some more cases of W_AbstractXxx but where the test already passes

diff --git a/pypy/objspace/std/test/test_intobject.py b/pypy/objspace/std/test/test_intobject.py
--- a/pypy/objspace/std/test/test_intobject.py
+++ b/pypy/objspace/std/test/test_intobject.py
@@ -684,6 +684,15 @@
         x = int(-sys.maxint)
         assert x.__rsub__(2) == (2 + sys.maxint)
 
+    def test_error_message_wrong_self(self):
+        unboundmeth = int.__str__
+        e = raises(TypeError, unboundmeth, "!")
+        assert "int" in str(e.value)
+        if hasattr(unboundmeth, 'im_func'):
+            e = raises(TypeError, unboundmeth.im_func, "!")
+            assert "'int'" in str(e.value)
+
+
 class AppTestIntShortcut(AppTestInt):
     spaceconfig = {"objspace.std.intshortcut": True}
 
diff --git a/pypy/objspace/std/test/test_longobject.py b/pypy/objspace/std/test/test_longobject.py
--- a/pypy/objspace/std/test/test_longobject.py
+++ b/pypy/objspace/std/test/test_longobject.py
@@ -450,3 +450,10 @@
         expected = (2 << (size * 4)) // 3
         assert long(n, 16) == expected
 
+    def test_error_message_wrong_self(self):
+        unboundmeth = long.__str__
+        e = raises(TypeError, unboundmeth, 42)
+        assert "long" in str(e.value)
+        if hasattr(unboundmeth, 'im_func'):
+            e = raises(TypeError, unboundmeth.im_func, 42)
+            assert "'long'" in str(e.value)
diff --git a/pypy/objspace/std/test/test_tupleobject.py b/pypy/objspace/std/test/test_tupleobject.py
--- a/pypy/objspace/std/test/test_tupleobject.py
+++ b/pypy/objspace/std/test/test_tupleobject.py
@@ -454,3 +454,11 @@
             (4.1, 2.3), (3.6, 4.8)]
         assert specialized_zip_2_lists(["foo", "bar"], [6, 2]) == [
             ("foo", 6), ("bar", 2)]
+
+    def test_error_message_wrong_self(self):
+        unboundmeth = tuple.__hash__
+        e = raises(TypeError, unboundmeth, 42)
+        assert "tuple" in str(e.value)
+        if hasattr(unboundmeth, 'im_func'):
+            e = raises(TypeError, unboundmeth.im_func, 42)
+            assert "'tuple'" in str(e.value)

From pypy.commits at gmail.com  Mon Oct 14 06:04:46 2019
From: pypy.commits at gmail.com (arigo)
Date: Mon, 14 Oct 2019 03:04:46 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: the merge of 'fix-descrmismatch-crash'
 into 'py3.6' will need this
Message-ID: <5da4483e.1c69fb81.5a1eb.3bbf@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.6
Changeset: r97776:3c57a384aae3
Date: 2019-10-14 12:04 +0200
http://bitbucket.org/pypy/pypy/changeset/3c57a384aae3/

Log:	the merge of 'fix-descrmismatch-crash' into 'py3.6' will need this
	line, not 'str'

diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -19,6 +19,7 @@
 
 class W_AbstractBytesObject(W_Root):
     __slots__ = ()
+    exact_class_applevel_name = 'bytes'
 
     def is_w(self, space, w_other):
         if not isinstance(w_other, W_AbstractBytesObject):

From pypy.commits at gmail.com  Mon Oct 14 12:11:10 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 14 Oct 2019 09:11:10 -0700 (PDT)
Subject: [pypy-commit] pypy default: Added tags for release-pypy2.7
Message-ID: <5da49e1e.1c69fb81.a4845.7e74@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r97777:c31b3fcf395f
Date: 2019-10-14 19:09 +0300
http://bitbucket.org/pypy/pypy/changeset/c31b3fcf395f/

Log:	Added tags for release-pypy2.7

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -55,3 +55,5 @@
 4d6761df14ffd6f38450f183ac1fad32c946c21b release-pypy3.6-v7.2.0rc1
 5da45ced70e515f94686be0df47c59abd1348ebc release-pypy3.6-v7.2.0rc2
 4a68d8d3d2fc1faec2e83bcb4d28559099092574 release-pypy2.7-v7.2.0rc2
+4a68d8d3d2fc1faec2e83bcb4d28559099092574 release-pypy2.7-v7.2.0
+5da45ced70e515f94686be0df47c59abd1348ebc release-pypy3.6-v7.2.0

From pypy.commits at gmail.com  Mon Oct 14 13:05:27 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 14 Oct 2019 10:05:27 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: update to 3.6.9 and update
 downloads for 7.2.0 release (ppc links commented out)
Message-ID: <5da4aad7.1c69fb81.1ad62.88c1@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: extradoc
Changeset: r954:4620c2cbcf08
Date: 2019-10-14 20:04 +0300
http://bitbucket.org/pypy/pypy.org/changeset/4620c2cbcf08/

Log:	update to 3.6.9 and update downloads for 7.2.0 release (ppc links
	commented out)

diff --git a/compat.html b/compat.html
--- a/compat.html
+++ b/compat.html
@@ -66,7 +66,7 @@
 language, passing Python test suite (with minor modifications that were
 already accepted in the main python in newer versions). It supports most
 of the commonly used Python <a class="reference external" href="http://docs.python.org/library/">standard library modules</a>; details below.</p>
-<p>PyPy3 implements the Python language version 3.5.3.  It has been released,
+<p>PyPy3 implements the Python language version 3.6.9.  It has been released,
 but Python is a large language and it is quite possible that a few things are missing.</p>
 <p class="download-menu"><a class="reference external" href="http://packages.pypy.org">List of installable top 1000 PyPI packages</a></p>
 <p>PyPy has support for the <a class="reference external" href="http://docs.python.org/c-api/">CPython C API</a>, however there are constructs
diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -66,11 +66,10 @@
 as stable as the release, but they contain numerous bugfixes and
 performance improvements.</p>
 <p>We provide binaries for x86, ARM, PPC and s390x running on different operating systems such as
-Linux, Mac OS X and Windows (<a class="reference external" href="http://doc.pypy.org/en/latest/release-v7.1.1.html">what's new in PyPy 7.1.1?</a>):</p>
+Linux, Mac OS X and Windows (<a class="reference external" href="http://doc.pypy.org/en/latest/release-v7.2.0.html">what's new in PyPy 7.2.0?</a>):</p>
 <ul class="simple">
-<li>the Python2.7 compatible release &#8212; <strong>PyPy2.7 v7.1.1</strong></li>
-<li>the Python3.6 compatible release, beta quality &#8212; <strong>PyPy3.6 v7.1.1</strong></li>
-<li>the (older) Python3.5 compatible release &#8212; <strong>PyPy3.5 v7.0</strong></li>
+<li>the Python2.7 compatible release &#8212; <strong>PyPy2.7 v7.2.0</strong></li>
+<li>the Python3.6 compatible release &#8212; <strong>PyPy3.6 v7.2.0</strong></li>
 <li>the Python2.7 Software Transactional Memory special release &#8212; <strong>PyPy-STM 2.5.1</strong> (Linux x86-64 only)</li>
 </ul>
 <ul class="download-menu simple">
@@ -108,55 +107,35 @@
 <li>or <a class="reference internal" href="#translate">translate</a> your own PyPy.</li>
 </ul>
 </div>
-<div class="section" id="python2-7-compatible-pypy-7-1-1">
-<span id="release"></span><h1>Python2.7 compatible PyPy 7.1.1</h1>
+<div class="section" id="python2-7-compatible-pypy-7-2-0">
+<span id="release"></span><h1>Python2.7 compatible PyPy 7.2.0</h1>
 <ul class="download-menu simple">
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.1-linux32.tar.bz2">Linux x86 binary (32bit, built on Ubuntu 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.1-linux64.tar.bz2">Linux x86-64 binary (64bit, built on Ubuntu 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.1-osx64.tar.bz2">Mac OS X binary (64bit)</a></li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.2.0-linux32.tar.bz2">Linux x86 binary (32bit, built on Ubuntu 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.2.0-linux64.tar.bz2">Linux x86-64 binary (64bit, built on Ubuntu 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.2.0-osx64.tar.bz2">Mac OS X binary (64bit)</a></li>
 <li>FreeBSD x86 and x86_64: see <a class="reference external" href="http://www.freshports.org/lang/pypy">FreshPorts</a></li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.1-win32.zip">Windows binary (32bit)</a> (you might need the VC runtime library
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.2.0-win32.zip">Windows binary (32bit)</a> (you might need the VC runtime library
 installer <a class="reference external" href="https://www.microsoft.com/en-us/download/details.aspx?id=52685">vcredist.x86.exe</a>.)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.1-ppc64.tar.bz2">PowerPC PPC64 Linux binary (64bit big-endian, Fedora 20)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.1-ppc64le.tar.bz2">PowerPC PPC64le Linux binary (64bit little-endian, Fedora 21)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.1-s390x.tar.bz2">s390x Linux binary (built on Redhat Linux 7.2)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.1-src.tar.bz2">Source (tar.bz2)</a>; <a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.1-src.zip">Source (zip)</a>.  See below for more about the sources.</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.2.0-aarch64.tar.bz2">Linux aarch64 binary (64bit, built on Ubuntu 18.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.2.0-s390x.tar.bz2">s390x Linux binary (built on Redhat Linux 7.2)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.2.0-src.tar.bz2">Source (tar.bz2)</a>; <a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.2.0-src.zip">Source (zip)</a>.  See below for more about the sources.</li>
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads">All our downloads,</a> including previous versions.  We also have a
 <a class="reference external" href="http://buildbot.pypy.org/mirror/">mirror</a>, but please use only if you have troubles accessing the links above</li>
-</ul>
-</div>
-<div class="section" id="python-3-6-compatible-pypy3-6-v7-1-1-beta">
-<h1>Python 3.6 compatible PyPy3.6 v7.1.1-beta</h1>
+</ul></div>
+<div class="section" id="python-3-6-compatible-pypy3-6-v7-2-0">
+<h1>Python 3.6 compatible PyPy3.6 v7.2.0</h1>
 <ul class="download-menu simple">
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.1-linux64.tar.bz2">Linux x86-64 binary (64bit, built on Ubuntu 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.1-linux32.tar.bz2">Linux x86 binary (32bit, built on Ubuntu 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.1-osx64.tar.bz2">Mac OS X binary (64bit)</a> (High Sierra &gt;= 10.13, not for Sierra and below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.1-win32.zip">Windows binary (32bit)</a> (you might need the VC runtime library
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.2.0-linux64.tar.bz2">Linux x86-64 binary (64bit, built on Ubuntu 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.2.0-linux32.tar.bz2">Linux x86 binary (32bit, built on Ubuntu 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.2.0-osx64.tar.bz2">Mac OS X binary (64bit)</a> (High Sierra &gt;= 10.13, not for Sierra and below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.2.0-win32.zip">Windows binary (32bit)</a> (you might need the VC runtime library
 installer <a class="reference external" href="https://www.microsoft.com/en-us/download/details.aspx?id=52685">vcredist.x86.exe</a>.)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.1-ppc64.tar.bz2">PowerPC PPC64 Linux binary (64bit big-endian, Fedora 20)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.1-ppc64le.tar.bz2">PowerPC PPC64le Linux binary (64bit little-endian, Fedora 21)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.1-s390x.tar.bz2">s390x Linux binary (built on Redhat Linux 7.2)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.1-src.tar.bz2">Source (tar.bz2)</a>; <a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.1-src.zip">Source (zip)</a>.  See below for more about the sources.</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.2.0-aarch64.tar.bz2">Linux aarch64 binary (64bit, built on Ubuntu 18.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.2.0-s390x.tar.bz2">s390x Linux binary (built on Redhat Linux 7.2)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.2.0-src.tar.bz2">Source (tar.bz2)</a>; <a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.2.0-src.zip">Source (zip)</a>.  See below for more about the sources.</li>
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads">All our downloads,</a> including previous versions.  We also have a
 <a class="reference external" href="http://buildbot.pypy.org/mirror/">mirror</a>, but please use only if you have troubles accessing the links above</li>
-</ul>
-</div>
-<div class="section" id="python-3-5-3-compatible-pypy3-5-v7-0-0">
-<h1>Python 3.5.3 compatible PyPy3.5 v7.0.0</h1>
-<ul class="download-menu simple">
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.5-v7.0.0-linux32.tar.bz2">Linux x86 binary (32bit, built on Ubuntu 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.5-v7.0.0-linux64.tar.bz2">Linux x86-64 binary (64bit, built on Ubuntu 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.5-v7.0.0-osx64.tar.bz2">Mac OS X binary (64bit)</a> (High Sierra &gt;= 10.13, not for Sierra and below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.5-v7.0.0-win32.zip">Windows binary (32bit)</a> <strong>BETA</strong> (you might need the VC runtime library
-installer <a class="reference external" href="https://www.microsoft.com/en-us/download/details.aspx?id=52685">vcredist.x86.exe</a>.)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.5-v7.0.0-ppc64.tar.bz2">PowerPC PPC64 Linux binary (64bit big-endian, Fedora 20)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.5-v7.0.0-ppc64le.tar.bz2">PowerPC PPC64le Linux binary (64bit little-endian, Fedora 21)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.5-v7.0.0-s390x.tar.bz2">s390x Linux binary (built on Redhat Linux 7.2)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.5-v7.0.0-src.tar.bz2">Source (tar.bz2)</a>; <a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.5-v7.0.0-src.zip">Source (zip)</a>.  See below for more about the sources.</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads">All our downloads,</a> including previous versions.  We also have a
-<a class="reference external" href="http://buildbot.pypy.org/mirror/">mirror</a>, but please use only if you have troubles accessing the links above</li>
-</ul>
-<p>If your CPU is really, really old, it may be a x86-32 without SSE2.
+</ul><p>If your CPU is really, really old, it may be a x86-32 without SSE2.
 There is untested support for manually translating PyPy's JIT without
 SSE2 (<tt class="docutils literal"><span class="pre">--jit-backend=x86-without-sse2</span></tt>) but note that your machine
 is probably low-spec enough that running CPython on it is a better
@@ -229,14 +208,14 @@
 </pre>
 <p>The trunk contains PyPy 2.  For PyPy 3, switch to the correct branch:</p>
 <pre class="literal-block">
-# for PyPy 3: switch to the branch of PyPy that implements Python 3.5
-hg update py3.5
+# for PyPy 3: switch to the branch of PyPy that implements Python 3.6
+hg update py3.6
 </pre>
 <p>Alternatively, get one of the following smaller packages for the source at
 the same revision as the above binaries:</p>
 <ul class="simple">
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.1-src.tar.bz2">pypy2.7-v7.1.1-src.tar.bz2</a> (sources, PyPy 2 only)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.1-src.tar.bz2">pypy3.6-v7.1.1-src.tar.bz2</a> (sources, PyPy 3 only)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.2.0-src.tar.bz2">pypy2.7-v7.2.0-src.tar.bz2</a> (sources, PyPy 2 only)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.2.0-src.tar.bz2">pypy3.6-v7.2.0-src.tar.bz2</a> (sources, PyPy 3 only)</li>
 </ul>
 </li>
 <li><p class="first">Make sure you <strong>installed the dependencies.</strong>  See the list <a class="reference external" href="http://pypy.readthedocs.org/en/latest/build.html#install-build-time-dependencies">here</a>.</p>
@@ -350,7 +329,18 @@
 </div>
 <div class="section" id="checksums">
 <h1>Checksums</h1>
-<p>Here are the checksums for each of the downloads of PyPy 7.1.1, 7.1.0, 7.0.0</p>
+<p>Here are the checksums for each of the downloads of PyPy 7.2.0, 7.1.1, 7.1.0</p>
+<p>pypy2.7-7.2.0 sha256:</p>
+<pre class="literal-block">
+57b0be053c6a5f069e23b843f38863cf7920f5eef7bc89f2e086e5c3a28a2ba9  pypy2.7-v7.2.0-aarch64.tar.bz2
+76d666e5aee54b519d6ec1af4ef0cbdc85f7f9276dd554e97deb026adfd0c936  pypy2.7-v7.2.0-linux32.tar.bz2
+05acf28e6a243026ecad933b9361d8f74b41f00818071b76b38c4694cc4c9599  pypy2.7-v7.2.0-linux64.tar.bz2
+36aa2f2440e762333569118dd0b3d5371d575c40966effa194d116c5453ddb52  pypy2.7-v7.2.0-osx64.tar.bz2
+bb7ae585ecb4d904c890e28a2c5b6bd379f57cc3d9e38ff45597ff54fa935eaa  pypy2.7-v7.2.0-s390x.tar.bz2
+55cb7757784fbe3952102447f65b27d80e6c885a464a7af1a9ce264492439dcc  pypy2.7-v7.2.0-src.tar.bz2
+897038550614d558f9f6718409b107e27903ef2b2b57ec250939d1b1ebdf0aba  pypy2.7-v7.2.0-src.zip
+956eeaaaac053e5d0917e77a3d2ad1933ab5561eb3e6e71235780b5aa5fd2bb7  pypy2.7-v7.2.0-win32.zip
+</pre>
 <p>pypy2.7-7.1.1 sha256:</p>
 <pre class="literal-block">
 41ca390a76ca0d47b8353a0d6a20d5aab5fad8b0bb647b960d8c33e873d18ef5  pypy2.7-v7.1.1-linux32.tar.bz2
@@ -371,6 +361,17 @@
 e60ce30f9947844da43daaa7658adc0c05330681305225954114772f42df06ec  pypy2.7-v7.1.0-src.zip
 76658c9ad679d562b8b6a09d006caa666406337b9834ff56db16980c5e549f20  pypy2.7-v7.1.0-win32.zip
 </pre>
+<p>pypy3.6-7.2.0 sha256:</p>
+<pre class="literal-block">
+f82dc9dc6c692417ee9727f23beae75364a5757ebdc657a2a1d0010ac3ad17ab  pypy3.6-v7.2.0-aarch64.tar.bz2
+45e99de197cb3e974cfc8d45e0076ad2066852e61e56b3eafd1237efafd2c43e  pypy3.6-v7.2.0-linux32.tar.bz2
+aa128e555ad0fe5c4c15104ae0903052bd232b6e3a73f5fe023d27b8fd0d6089  pypy3.6-v7.2.0-linux64.tar.bz2
+836abb0ec303b90a684533711ed3b8269d3e8c64805b595e410920abdea678ac  pypy3.6-v7.2.0-osx64.tar.bz2
+a11da8118064db102d159e9221319c428b298c4a87f26166fd6ae94be8d6ae0d  pypy3.6-v7.2.0-s390x.tar.bz2
+0d7c707df5041f1593fe82f29c40056c21e4d6cb66554bbd66769bd80bcbfafc  pypy3.6-v7.2.0-src.tar.bz2
+405ac35695dd374d5ea192cb44cb47231f9a65812cc7b6549df33df12ffe54db  pypy3.6-v7.2.0-src.zip
+c926f622bec24a8b348591d631717ace83b3a6c3c2dac02b157b622b97d1fc9c  pypy3.6-v7.2.0-win32.zip
+</pre>
 <p>pypy3.6-7.1.1 sha256:</p>
 <pre class="literal-block">
 cb11ef4b0df569c28390b1ee93029159e1b90bfbad98df6abd629d5203b2abd9  pypy3.6-v7.1.1-linux32.tar.bz2
@@ -403,18 +404,6 @@
 f51d8bbfc4e73a8a01820b7871a45d13c59f1399822cdf8a19388c69eb20c18c  pypy2.7-v7.0.0-src.tar.bz2
 77c8c02cf412a5f8182ffe8845877cffa506e5a5ce3a7cd835483fdc1202afd4  pypy2.7-v7.0.0-src.zip
 </pre>
-<p>pypy 3.5-v7.0.0 sha256:</p>
-<pre class="literal-block">
-b8db8fbca9621de8ea8cd7184b322f2dddb2f385e8e5a63dfb75bb3fea4b2e3f  pypy3.5-v7.0.0-linux32.tar.bz2
-729e3c54325969c98bd3658c6342b9f5987b96bad1d6def04250a08401b54c4b  pypy3.5-v7.0.0-linux64.tar.bz2
-7c6d71653d9b1a7946d1eeebbf24b454fe934fba8b0c39f648bdc545fb2895ce  pypy3.5-v7.0.0-osx64.tar.bz2
-d588b045cc0d3a75c31fce54c1d181b1206ad9a5dd272fe79160a6268401605f  pypy3.5-v7.0.0-s390x.tar.bz2
-23e30b00ab61f24578059e4643fbf0221982faffd874898b5737fc5b334ca0ab  pypy3.5-v7.0.0-ppc64.tar.bz2
-2912884da05abc2cdf71dd337c3f280095351312c1a1732a52b6878174a0fd02  pypy3.5-v7.0.0-ppc64le.tar.bz2
-TODO win32
-b2ddb0f45cb4e0384fb498ef7fcca2ac96c730b9000affcf8d730169397f017f  pypy3.5-v7.0.0-src.tar.bz2
-3aa3a921c163667761165dbd2070e56d6715979fe9cc1f135d58ea0692a05a1e  pypy3.5-v7.0.0-src.zip
-</pre>
 </div>
 </div>
 <div id="sidebar">
diff --git a/features.html b/features.html
--- a/features.html
+++ b/features.html
@@ -65,16 +65,16 @@
 <p>PyPy is a replacement for CPython.  It is built using the RPython
 language that was co-developed with it.  The main reason to use it
 instead of CPython is speed: it runs generally faster (see next section).</p>
-<p><strong>PyPy</strong> implements <strong>Python 2.7.13 and 3.5.3</strong>.
-It supports all of the core language, passing the Python test suite
-(with minor modifications that were already accepted in the main python
-in newer versions). It supports most of the commonly used Python
-standard library modules.  For known differences with CPython, see our
-<a class="reference external" href="compat.html">compatibility</a> page.</p>
+<p><strong>PyPy</strong> implements <strong>Python 2.7.13 and 3.6.9</strong>.
+It supports all of the core language, passing the Python 2.7 test suite
+and most of the 3.6 test suite (with minor modifications) It supports most of
+the commonly used Python standard library modules. For known differences with
+CPython, see our <a class="reference external" href="compat.html">compatibility</a> page.</p>
 <p>The following CPU architectures are supported and maintained:</p>
 <ul class="simple">
 <li><a class="reference external" href="http://en.wikipedia.org/wiki/IA-32">x86 (IA-32)</a> and <a class="reference external" href="http://en.wikipedia.org/wiki/X86_64">x86_64</a></li>
 <li><a class="reference external" href="http://en.wikipedia.org/wiki/ARM">ARM</a> platforms (ARMv6 or ARMv7, with VFPv3)</li>
+<li><a class="reference external" href="http://en.wikipedia.org/wiki/AArch64">AArch64</a></li>
 <li><a class="reference external" href="https://de.wikipedia.org/wiki/PowerPC">PowerPC</a> 64bit both little and big endian</li>
 <li><a class="reference external" href="https://de.wikipedia.org/wiki/System/390">System Z (s390x)</a></li>
 </ul>
diff --git a/index.html b/index.html
--- a/index.html
+++ b/index.html
@@ -63,7 +63,7 @@
 <div id="main">
 <h1 class="title">Welcome to PyPy</h1>
 <p>PyPy is a <a class="reference external" href="http://speed.pypy.org/">fast</a>, <a class="reference external" href="compat.html">compliant</a> alternative implementation of the <a class="reference external" href="http://python.org/">Python</a>
-language (2.7.13 and 3.5.3, 3.6). It has several advantages and distinct features:</p>
+language (2.7.13 and 3.6.9). It has several advantages and distinct features:</p>
 <blockquote>
 <ul>
 <li><p class="first"><strong>Speed:</strong> thanks to its Just-in-Time compiler, Python programs
diff --git a/source/compat.txt b/source/compat.txt
--- a/source/compat.txt
+++ b/source/compat.txt
@@ -8,7 +8,7 @@
 already accepted in the main python in newer versions). It supports most
 of the commonly used Python `standard library modules`_; details below.
 
-PyPy3 implements the Python language version 3.5.3.  It has been released,
+PyPy3 implements the Python language version 3.6.9.  It has been released,
 but Python is a large language and it is quite possible that a few things are missing.
 
 .. class:: download_menu
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -13,17 +13,15 @@
   performance improvements.
 
 We provide binaries for x86, ARM, PPC and s390x running on different operating systems such as
-Linux, Mac OS X and Windows (`what's new in PyPy 7.1.1?`_):
+Linux, Mac OS X and Windows (`what's new in PyPy 7.2.0?`_):
 
-* the Python2.7 compatible release &#8212; **PyPy2.7 v7.1.1**
+* the Python2.7 compatible release &#8212; **PyPy2.7 v7.2.0**
 
-* the Python3.6 compatible release, beta quality &#8212; **PyPy3.6 v7.1.1**
-
-* the (older) Python3.5 compatible release &#8212; **PyPy3.5 v7.0**
+* the Python3.6 compatible release &#8212; **PyPy3.6 v7.2.0**
 
 * the Python2.7 Software Transactional Memory special release &#8212; **PyPy-STM 2.5.1** (Linux x86-64 only)
 
-.. _what's new in PyPy 7.1.1?: http://doc.pypy.org/en/latest/release-v7.1.1.html
+.. _what's new in PyPy 7.2.0?: http://doc.pypy.org/en/latest/release-v7.2.0.html
 
 
 .. class:: download_menu
@@ -79,7 +77,7 @@
 
 .. _release:
 
-Python2.7 compatible PyPy 7.1.1
+Python2.7 compatible PyPy 7.2.0
 -------------------------------
 
 .. class:: download_menu
@@ -90,30 +88,31 @@
 * FreeBSD x86 and x86_64: see FreshPorts_
 * `Windows binary (32bit)`__ (you might need the VC runtime library
   installer `vcredist.x86.exe`_.)
-* `PowerPC PPC64 Linux binary (64bit big-endian, Fedora 20)`__ (see ``[1]`` below)
-* `PowerPC PPC64le Linux binary (64bit little-endian, Fedora 21)`__ (see ``[1]`` below)
+* `Linux aarch64 binary (64bit, built on Ubuntu 18.04)`__ (see ``[1]`` below)
 * `s390x Linux binary (built on Redhat Linux 7.2)`__ (see ``[1]`` below)
 * `Source (tar.bz2)`__; `Source (zip)`__.  See below for more about the sources.
 * `All our downloads,`__ including previous versions.  We also have a
   mirror_, but please use only if you have troubles accessing the links above
 
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.1-linux32.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.1-linux64.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.1-osx64.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.1-win32.zip
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.1-ppc64.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.1-ppc64le.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.1-s390x.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.1-src.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.1-src.zip
+.. * `PowerPC PPC64 Linux binary (64bit big-endian, Fedora 20)`__ (see ``[1]`` below)
+.. * `PowerPC PPC64le Linux binary (64bit little-endian, Fedora 21)`__ (see ``[1]`` below)
+
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.2.0-linux32.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.2.0-linux64.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.2.0-osx64.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.2.0-win32.zip
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.2.0-aarch64.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.2.0-s390x.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.2.0-src.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.2.0-src.zip
 .. _`vcredist.x86.exe`: https://www.microsoft.com/en-us/download/details.aspx?id=52685
 .. __: https://bitbucket.org/pypy/pypy/downloads
 .. _mirror: http://buildbot.pypy.org/mirror/
 .. _FreshPorts: http://www.freshports.org/lang/pypy
 
 
-Python 3.6 compatible PyPy3.6 v7.1.1-beta
------------------------------------------
+Python 3.6 compatible PyPy3.6 v7.2.0
+------------------------------------
 
 .. class:: download_menu
 
@@ -122,51 +121,23 @@
 * `Mac OS X binary (64bit)`__ (High Sierra >= 10.13, not for Sierra and below)
 * `Windows binary (32bit)`__ (you might need the VC runtime library
   installer `vcredist.x86.exe`_.)
-* `PowerPC PPC64 Linux binary (64bit big-endian, Fedora 20)`__ (see ``[1]`` below)
-* `PowerPC PPC64le Linux binary (64bit little-endian, Fedora 21)`__ (see ``[1]`` below)
+* `Linux aarch64 binary (64bit, built on Ubuntu 18.04)`__ (see ``[1]`` below)
 * `s390x Linux binary (built on Redhat Linux 7.2)`__ (see ``[1]`` below)
 * `Source (tar.bz2)`__; `Source (zip)`__.  See below for more about the sources.
 * `All our downloads,`__ including previous versions.  We also have a
   mirror_, but please use only if you have troubles accessing the links above
 
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.1-linux64.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.1-linux32.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.1-osx64.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.1-win32.zip
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.1-ppc64.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.1-ppc64le.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.1-s390x.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.1-src.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.1-src.zip
-.. __: https://bitbucket.org/pypy/pypy/downloads
+.. * `PowerPC PPC64 Linux binary (64bit big-endian, Fedora 20)`__ (see ``[1]`` below)
+.. * `PowerPC PPC64le Linux binary (64bit little-endian, Fedora 21)`__ (see ``[1]`` below)
 
-
-Python 3.5.3 compatible PyPy3.5 v7.0.0
----------------------------------------
-
-.. class:: download_menu
-
-* `Linux x86 binary (32bit, built on Ubuntu 14.04)`__ (see ``[1]`` below)
-* `Linux x86-64 binary (64bit, built on Ubuntu 14.04)`__ (see ``[1]`` below)
-* `Mac OS X binary (64bit)`__ (High Sierra >= 10.13, not for Sierra and below)
-* `Windows binary (32bit)`__ **BETA** (you might need the VC runtime library
-  installer `vcredist.x86.exe`_.)
-* `PowerPC PPC64 Linux binary (64bit big-endian, Fedora 20)`__ (see ``[1]`` below)
-* `PowerPC PPC64le Linux binary (64bit little-endian, Fedora 21)`__ (see ``[1]`` below)
-* `s390x Linux binary (built on Redhat Linux 7.2)`__ (see ``[1]`` below)
-* `Source (tar.bz2)`__; `Source (zip)`__.  See below for more about the sources.
-* `All our downloads,`__ including previous versions.  We also have a
-  mirror_, but please use only if you have troubles accessing the links above
-
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.5-v7.0.0-linux32.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.5-v7.0.0-linux64.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.5-v7.0.0-osx64.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.5-v7.0.0-win32.zip
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.5-v7.0.0-ppc64.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.5-v7.0.0-ppc64le.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.5-v7.0.0-s390x.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.5-v7.0.0-src.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.5-v7.0.0-src.zip
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.2.0-linux64.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.2.0-linux32.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.2.0-osx64.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.2.0-win32.zip
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.2.0-aarch64.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.2.0-s390x.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.2.0-src.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.2.0-src.zip
 .. __: https://bitbucket.org/pypy/pypy/downloads
 
 
@@ -268,17 +239,17 @@
 
    The trunk contains PyPy 2.  For PyPy 3, switch to the correct branch::
 
-     # for PyPy 3: switch to the branch of PyPy that implements Python 3.5
-     hg update py3.5
+     # for PyPy 3: switch to the branch of PyPy that implements Python 3.6
+     hg update py3.6
 
    Alternatively, get one of the following smaller packages for the source at
    the same revision as the above binaries:
 
-   * `pypy2.7-v7.1.1-src.tar.bz2`__ (sources, PyPy 2 only)
-   * `pypy3.6-v7.1.1-src.tar.bz2`__ (sources, PyPy 3 only)
+   * `pypy2.7-v7.2.0-src.tar.bz2`__ (sources, PyPy 2 only)
+   * `pypy3.6-v7.2.0-src.tar.bz2`__ (sources, PyPy 3 only)
 
-   .. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.1-src.tar.bz2
-   .. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.1-src.tar.bz2
+   .. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.2.0-src.tar.bz2
+   .. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.2.0-src.tar.bz2
 
 
 2. Make sure you **installed the dependencies.**  See the list here__.
@@ -406,7 +377,18 @@
 Checksums
 ---------
 
-Here are the checksums for each of the downloads of PyPy 7.1.1, 7.1.0, 7.0.0
+Here are the checksums for each of the downloads of PyPy 7.2.0, 7.1.1, 7.1.0
+
+pypy2.7-7.2.0 sha256::
+
+    57b0be053c6a5f069e23b843f38863cf7920f5eef7bc89f2e086e5c3a28a2ba9  pypy2.7-v7.2.0-aarch64.tar.bz2
+    76d666e5aee54b519d6ec1af4ef0cbdc85f7f9276dd554e97deb026adfd0c936  pypy2.7-v7.2.0-linux32.tar.bz2
+    05acf28e6a243026ecad933b9361d8f74b41f00818071b76b38c4694cc4c9599  pypy2.7-v7.2.0-linux64.tar.bz2
+    36aa2f2440e762333569118dd0b3d5371d575c40966effa194d116c5453ddb52  pypy2.7-v7.2.0-osx64.tar.bz2
+    bb7ae585ecb4d904c890e28a2c5b6bd379f57cc3d9e38ff45597ff54fa935eaa  pypy2.7-v7.2.0-s390x.tar.bz2
+    55cb7757784fbe3952102447f65b27d80e6c885a464a7af1a9ce264492439dcc  pypy2.7-v7.2.0-src.tar.bz2
+    897038550614d558f9f6718409b107e27903ef2b2b57ec250939d1b1ebdf0aba  pypy2.7-v7.2.0-src.zip
+    956eeaaaac053e5d0917e77a3d2ad1933ab5561eb3e6e71235780b5aa5fd2bb7  pypy2.7-v7.2.0-win32.zip
 
 pypy2.7-7.1.1 sha256::
 
@@ -428,6 +410,17 @@
     e60ce30f9947844da43daaa7658adc0c05330681305225954114772f42df06ec  pypy2.7-v7.1.0-src.zip
     76658c9ad679d562b8b6a09d006caa666406337b9834ff56db16980c5e549f20  pypy2.7-v7.1.0-win32.zip
 
+pypy3.6-7.2.0 sha256::
+
+    f82dc9dc6c692417ee9727f23beae75364a5757ebdc657a2a1d0010ac3ad17ab  pypy3.6-v7.2.0-aarch64.tar.bz2
+    45e99de197cb3e974cfc8d45e0076ad2066852e61e56b3eafd1237efafd2c43e  pypy3.6-v7.2.0-linux32.tar.bz2
+    aa128e555ad0fe5c4c15104ae0903052bd232b6e3a73f5fe023d27b8fd0d6089  pypy3.6-v7.2.0-linux64.tar.bz2
+    836abb0ec303b90a684533711ed3b8269d3e8c64805b595e410920abdea678ac  pypy3.6-v7.2.0-osx64.tar.bz2
+    a11da8118064db102d159e9221319c428b298c4a87f26166fd6ae94be8d6ae0d  pypy3.6-v7.2.0-s390x.tar.bz2
+    0d7c707df5041f1593fe82f29c40056c21e4d6cb66554bbd66769bd80bcbfafc  pypy3.6-v7.2.0-src.tar.bz2
+    405ac35695dd374d5ea192cb44cb47231f9a65812cc7b6549df33df12ffe54db  pypy3.6-v7.2.0-src.zip
+    c926f622bec24a8b348591d631717ace83b3a6c3c2dac02b157b622b97d1fc9c  pypy3.6-v7.2.0-win32.zip
+
 pypy3.6-7.1.1 sha256::
 
     cb11ef4b0df569c28390b1ee93029159e1b90bfbad98df6abd629d5203b2abd9  pypy3.6-v7.1.1-linux32.tar.bz2
@@ -461,18 +454,3 @@
     f51d8bbfc4e73a8a01820b7871a45d13c59f1399822cdf8a19388c69eb20c18c  pypy2.7-v7.0.0-src.tar.bz2
     77c8c02cf412a5f8182ffe8845877cffa506e5a5ce3a7cd835483fdc1202afd4  pypy2.7-v7.0.0-src.zip
 
-
-pypy 3.5-v7.0.0 sha256::
-
-    b8db8fbca9621de8ea8cd7184b322f2dddb2f385e8e5a63dfb75bb3fea4b2e3f  pypy3.5-v7.0.0-linux32.tar.bz2
-    729e3c54325969c98bd3658c6342b9f5987b96bad1d6def04250a08401b54c4b  pypy3.5-v7.0.0-linux64.tar.bz2
-    7c6d71653d9b1a7946d1eeebbf24b454fe934fba8b0c39f648bdc545fb2895ce  pypy3.5-v7.0.0-osx64.tar.bz2
-    d588b045cc0d3a75c31fce54c1d181b1206ad9a5dd272fe79160a6268401605f  pypy3.5-v7.0.0-s390x.tar.bz2
-    23e30b00ab61f24578059e4643fbf0221982faffd874898b5737fc5b334ca0ab  pypy3.5-v7.0.0-ppc64.tar.bz2
-    2912884da05abc2cdf71dd337c3f280095351312c1a1732a52b6878174a0fd02  pypy3.5-v7.0.0-ppc64le.tar.bz2
-    
-    TODO win32
-
-    b2ddb0f45cb4e0384fb498ef7fcca2ac96c730b9000affcf8d730169397f017f  pypy3.5-v7.0.0-src.tar.bz2
-    3aa3a921c163667761165dbd2070e56d6715979fe9cc1f135d58ea0692a05a1e  pypy3.5-v7.0.0-src.zip
-
diff --git a/source/features.txt b/source/features.txt
--- a/source/features.txt
+++ b/source/features.txt
@@ -10,17 +10,17 @@
 language that was co-developed with it.  The main reason to use it
 instead of CPython is speed: it runs generally faster (see next section).
 
-**PyPy** implements **Python 2.7.13 and 3.5.3**.
-It supports all of the core language, passing the Python test suite
-(with minor modifications that were already accepted in the main python
-in newer versions). It supports most of the commonly used Python
-standard library modules.  For known differences with CPython, see our
-`compatibility`_ page.
+**PyPy** implements **Python 2.7.13 and 3.6.9**.
+It supports all of the core language, passing the Python 2.7 test suite
+and most of the 3.6 test suite (with minor modifications) It supports most of
+the commonly used Python standard library modules. For known differences with
+CPython, see our `compatibility`_ page.
 
 The following CPU architectures are supported and maintained:
 
 * `x86 (IA-32)`_ and `x86_64`_ 
 * `ARM`_ platforms (ARMv6 or ARMv7, with VFPv3)
+* `AArch64`_
 * `PowerPC`_ 64bit both little and big endian
 * `System Z (s390x)`_
 
@@ -34,6 +34,7 @@
 .. _`x86 (IA-32)`: http://en.wikipedia.org/wiki/IA-32
 .. _`x86_64`: http://en.wikipedia.org/wiki/X86_64
 .. _`ARM`: http://en.wikipedia.org/wiki/ARM
+.. _`AArch64`: http://en.wikipedia.org/wiki/AArch64
 .. _`PowerPC`: https://de.wikipedia.org/wiki/PowerPC
 .. _`System Z (s390x)`: https://de.wikipedia.org/wiki/System/390
 .. _`howtohelp`: howtohelp.html
diff --git a/source/index.txt b/source/index.txt
--- a/source/index.txt
+++ b/source/index.txt
@@ -4,7 +4,7 @@
 ---
 
 PyPy is a `fast`_, `compliant`_ alternative implementation of the `Python`_
-language (2.7.13 and 3.5.3, 3.6). It has several advantages and distinct features:
+language (2.7.13 and 3.6.9). It has several advantages and distinct features:
 
   * **Speed:** thanks to its Just-in-Time compiler, Python programs
     often run `faster`_ on PyPy.  `(What is a JIT compiler?)`_

From pypy.commits at gmail.com  Mon Oct 14 14:44:07 2019
From: pypy.commits at gmail.com (arigo)
Date: Mon, 14 Oct 2019 11:44:07 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: Clarify that this is not the new
 sandboxing but the old one
Message-ID: <5da4c1f7.1c69fb81.ad419.2733@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r956:b19fc2740663
Date: 2019-10-14 20:43 +0200
http://bitbucket.org/pypy/pypy.org/changeset/b19fc2740663/

Log:	Clarify that this is not the new sandboxing but the old one

diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -172,7 +172,9 @@
 documentation</a>.</li>
 </ul>
 <ul class="simple">
-<li>Sandboxing: A special safe version.  Read the docs about <a class="reference external" href="features.html#sandboxing">sandboxing</a>.
+<li>Old-style sandboxing: A special safe version.
+<em>This is NOT the version announced in-development during 2019!</em>
+Read the docs about <a class="reference external" href="features.html#sandboxing">sandboxing</a>.
 This version is <strong>not supported</strong> and not actively maintained.  You
 will likely have to fix some issues yourself, or checkout an old
 version, or otherwise play around on your own.  We provide this
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -185,7 +185,9 @@
 
 .. __: https://bitbucket.org/pypy/revdb/
 
-* Sandboxing: A special safe version.  Read the docs about sandboxing_.
+* Old-style sandboxing: A special safe version.
+  *This is NOT the version announced in-development during 2019!*
+  Read the docs about sandboxing_.
   This version is **not supported** and not actively maintained.  You
   will likely have to fix some issues yourself, or checkout an old
   version, or otherwise play around on your own.  We provide this

From pypy.commits at gmail.com  Tue Oct 15 03:48:51 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 15 Oct 2019 00:48:51 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: fix linux32 build platform,
 it is actually 16.04
Message-ID: <5da579e3.1c69fb81.4e610.7c4b@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: extradoc
Changeset: r957:c8638412b8ba
Date: 2019-10-15 10:48 +0300
http://bitbucket.org/pypy/pypy.org/changeset/c8638412b8ba/

Log:	fix linux32 build platform, it is actually 16.04

diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -110,7 +110,7 @@
 <div class="section" id="python2-7-compatible-pypy-7-2-0">
 <span id="release"></span><h1>Python2.7 compatible PyPy 7.2.0</h1>
 <ul class="download-menu simple">
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.2.0-linux32.tar.bz2">Linux x86 binary (32bit, built on Ubuntu 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.2.0-linux32.tar.bz2">Linux x86 binary (32bit, built on Ubuntu 16.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.2.0-linux64.tar.bz2">Linux x86-64 binary (64bit, built on Ubuntu 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.2.0-osx64.tar.bz2">Mac OS X binary (64bit)</a></li>
 <li>FreeBSD x86 and x86_64: see <a class="reference external" href="http://www.freshports.org/lang/pypy">FreshPorts</a></li>
@@ -128,7 +128,7 @@
 <div class="section" id="python-3-6-compatible-pypy3-6-v7-2-0">
 <h1>Python 3.6 compatible PyPy3.6 v7.2.0</h1>
 <ul class="download-menu simple">
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.2.0-linux64.tar.bz2">Linux x86-64 binary (64bit, built on Ubuntu 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.2.0-linux64.tar.bz2">Linux x86-64 binary (64bit, built on Ubuntu 16.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.2.0-linux32.tar.bz2">Linux x86 binary (32bit, built on Ubuntu 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.2.0-osx64.tar.bz2">Mac OS X binary (64bit)</a> (High Sierra &gt;= 10.13, not for Sierra and below)</li>
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.2.0-win32.zip">Windows binary (32bit)</a> (you might need the VC runtime library
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -82,7 +82,7 @@
 
 .. class:: download_menu
 
-* `Linux x86 binary (32bit, built on Ubuntu 14.04)`__ (see ``[1]`` below)
+* `Linux x86 binary (32bit, built on Ubuntu 16.04)`__ (see ``[1]`` below)
 * `Linux x86-64 binary (64bit, built on Ubuntu 14.04)`__ (see ``[1]`` below)
 * `Mac OS X binary (64bit)`__
 * FreeBSD x86 and x86_64: see FreshPorts_
@@ -118,7 +118,7 @@
 
 .. class:: download_menu
 
-* `Linux x86-64 binary (64bit, built on Ubuntu 14.04)`__ (see ``[1]`` below)
+* `Linux x86-64 binary (64bit, built on Ubuntu 16.04)`__ (see ``[1]`` below)
 * `Linux x86 binary (32bit, built on Ubuntu 14.04)`__ (see ``[1]`` below)
 * `Mac OS X binary (64bit)`__ (High Sierra >= 10.13, not for Sierra and below)
 * `Windows binary (32bit)`__ (you might need the VC runtime library

From pypy.commits at gmail.com  Tue Oct 15 05:24:30 2019
From: pypy.commits at gmail.com (arigo)
Date: Tue, 15 Oct 2019 02:24:30 -0700 (PDT)
Subject: [pypy-commit] cffi default: Windows fix: sometimes,
 time.sleep() doesn't sleep apparently
Message-ID: <5da5904e.1c69fb81.b7af.0eec@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3295:52552b435eb5
Date: 2019-10-15 11:24 +0200
http://bitbucket.org/cffi/cffi/changeset/52552b435eb5/

Log:	Windows fix: sometimes, time.sleep() doesn't sleep apparently

diff --git a/testing/embedding/add1.py b/testing/embedding/add1.py
--- a/testing/embedding/add1.py
+++ b/testing/embedding/add1.py
@@ -11,7 +11,11 @@
     sys.stdout.write("preparing")
     for i in range(3):
         sys.stdout.flush()
-        time.sleep(0.2)
+        # Windows: sometimes time.sleep() doesn't sleep at all.
+        # This appears to occur on recent versions of python only.
+        t_end = time.time() + 0.19
+        while time.time() < t_end:
+            time.sleep(0.2)
         sys.stdout.write(".")
     sys.stdout.write("\n")
 
diff --git a/testing/embedding/test_thread.py b/testing/embedding/test_thread.py
--- a/testing/embedding/test_thread.py
+++ b/testing/embedding/test_thread.py
@@ -21,17 +21,21 @@
         add1_cffi = self.prepare_module('add1')
         add2_cffi = self.prepare_module('add2')
         self.compile('thread2-test', [add1_cffi, add2_cffi], threads=True)
-        output = self.execute('thread2-test')
-        output = self._take_out(output, "preparing")
-        output = self._take_out(output, ".")
-        output = self._take_out(output, ".")
-        # at least the 3rd dot should be after everything from ADD2
-        assert output == ("starting\n"
-                          "prepADD2\n"
-                          "adding 1000 and 200 and 30\n"
-                          ".\n"
-                          "adding 40 and 2\n"
-                          "done\n")
+        for i in range(3):
+            output = self.execute('thread2-test')
+            print('='*79)
+            print(output)
+            print('='*79)
+            output = self._take_out(output, "preparing")
+            output = self._take_out(output, ".")
+            output = self._take_out(output, ".")
+            # at least the 3rd dot should be after everything from ADD2
+            assert output == ("starting\n"
+                              "prepADD2\n"
+                              "adding 1000 and 200 and 30\n"
+                              ".\n"
+                              "adding 40 and 2\n"
+                              "done\n")
 
     def test_alt_issue(self):
         add1_cffi = self.prepare_module('add1')

From pypy.commits at gmail.com  Tue Oct 15 08:27:06 2019
From: pypy.commits at gmail.com (arigo)
Date: Tue, 15 Oct 2019 05:27:06 -0700 (PDT)
Subject: [pypy-commit] cffi release-1.13: md5/sha
Message-ID: <5da5bb1a.1c69fb81.74b51.08e9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-1.13
Changeset: r3296:17c06e75f54c
Date: 2019-10-15 14:26 +0200
http://bitbucket.org/cffi/cffi/changeset/17c06e75f54c/

Log:	md5/sha

diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -54,11 +54,11 @@
 
 * Checksums of the "source" package version 1.13.0:
 
-   - MD5: ...
+   - MD5: 2c90be6d3bfbf80ab732b28fd677051d
 
-   - SHA: ...
+   - SHA: 75977bbad8ea8c15f2c34edde5efcf39462ea43e
 
-   - SHA256: ...
+   - SHA256: 8fe230f612c18af1df6f348d02d682fe2c28ca0a6c3856c99599cdacae7cf226
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``

From pypy.commits at gmail.com  Tue Oct 15 08:27:08 2019
From: pypy.commits at gmail.com (arigo)
Date: Tue, 15 Oct 2019 05:27:08 -0700 (PDT)
Subject: [pypy-commit] cffi release-1.13: Added tag v1.13.0 for changeset
 17c06e75f54c
Message-ID: <5da5bb1c.1c69fb81.a7370.509f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-1.13
Changeset: r3297:e5716858c2c5
Date: 2019-10-15 14:26 +0200
http://bitbucket.org/cffi/cffi/changeset/e5716858c2c5/

Log:	Added tag v1.13.0 for changeset 17c06e75f54c

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -21,3 +21,4 @@
 ac7cb142e8f5dc73033d89c54c54e1b1a6413946 v1.12.1
 e0c76668ecf5ab1ad005799cfe1b7cb58610f155 v1.12.2
 bf80722dea36237710083315e336c81bc8571fd0 v1.12.3
+17c06e75f54cf5a31865961f0b498f99232a0bc8 v1.13.0

From pypy.commits at gmail.com  Tue Oct 15 08:27:09 2019
From: pypy.commits at gmail.com (arigo)
Date: Tue, 15 Oct 2019 05:27:09 -0700 (PDT)
Subject: [pypy-commit] cffi default: hg merge release-1.13.0
Message-ID: <5da5bb1d.1c69fb81.ceec.b3dd@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3298:068770676b43
Date: 2019-10-15 14:26 +0200
http://bitbucket.org/cffi/cffi/changeset/068770676b43/

Log:	hg merge release-1.13.0

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -21,3 +21,4 @@
 ac7cb142e8f5dc73033d89c54c54e1b1a6413946 v1.12.1
 e0c76668ecf5ab1ad005799cfe1b7cb58610f155 v1.12.2
 bf80722dea36237710083315e336c81bc8571fd0 v1.12.3
+17c06e75f54cf5a31865961f0b498f99232a0bc8 v1.13.0
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -54,11 +54,11 @@
 
 * Checksums of the "source" package version 1.13.0:
 
-   - MD5: ...
+   - MD5: 2c90be6d3bfbf80ab732b28fd677051d
 
-   - SHA: ...
+   - SHA: 75977bbad8ea8c15f2c34edde5efcf39462ea43e
 
-   - SHA256: ...
+   - SHA256: 8fe230f612c18af1df6f348d02d682fe2c28ca0a6c3856c99599cdacae7cf226
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``

From pypy.commits at gmail.com  Tue Oct 15 08:39:59 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 15 Oct 2019 05:39:59 -0700 (PDT)
Subject: [pypy-commit] buildbot default: update linux64 chroot description,
 try to fix aarch64 view in nightlies
Message-ID: <5da5be1f.1c69fb81.784e4.6939@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r1096:e2b202b96f3e
Date: 2019-10-15 15:39 +0300
http://bitbucket.org/pypy/buildbot/changeset/e2b202b96f3e/

Log:	update linux64 chroot description, try to fix aarch64 view in
	nightlies

diff --git a/README-CHROOT b/README-CHROOT
--- a/README-CHROOT
+++ b/README-CHROOT
@@ -2,53 +2,62 @@
 ---------------------------------------------------------
 Buildslave runs in a chroot:
 
-    LANG=C chroot /mnt/ubuntu /bin/bash
+    LANG=C chroot /extra1/xenial64 /bin/bash
     su - buildslave
-
+    buildslave start /home/buildslave/build_dir
 
 Commands to run once after a reboot to set the chroot:
 
     >>> have been copied into /etc/rc.local
 
-mount --rbind /tmp /home/tmp
-mount --rbind /extra1 /home/extra1
-mount --rbind /proc /mnt/ubuntu/proc
-mount --rbind /sys /mnt/ubuntu/sys
-mount --rbind /dev /mnt/ubuntu/dev
-mount --rbind /run/lock /mnt/ubuntu/run/lock
-mount --rbind /run/shm /mnt/ubuntu/run/shm
-
+# from the older 14.04 chroot on /mnt/ubuntu
+#mount --rbind /tmp /home/tmp
+#mount --rbind /extra1 /home/extra1
+#mount --rbind /proc /mnt/ubuntu/proc
+#mount --rbind /sys /mnt/ubuntu/sys
+#mount --rbind /dev /mnt/ubuntu/dev
+#mount --rbind /run/lock /mnt/ubuntu/run/lock
+#mount --rbind /run/shm /mnt/ubuntu/run/shm
+# now use /extra1/xenial64
+mount --bind /tmp /extra1/xenial64/tmp
+mount --rbind /proc /extra1/xenial64/proc
+mount --rbind /sys /extra1/xenial64/sys
+mount --rbind /dev /extra1/xenial64/dev
+mount --rbind /run/lock /extra1/xenial64/run/lock
+mount --rbind /run/shm /extra1/xenial64/run/shm
 
 ---------------------------------------------------------
 And here is the same for the 32-bit chroot
 ---------------------------------------------------------
-32 bit buildslave runs in a debian strech chroot,
-chosen since debian comes with gcc6:
+32 bit buildslave runs in a ubuntu xenial chroot on benchmarker:
 
-    LANG=C chroot /extra1/stretch32 /bin/bash
+    LANG=C chroot /opt/xenial32 /bin/bash
     su - buildslave
 
 Commands to run once after a reboot to set the chroot:
 
     >>> have been copied into /etc/rc.local
-mount --rbind /tmp /extra1/stretch32/tmp
-mount --rbind /proc /extra1/stretch32/proc
-mount --rbind /sys /extra1/stretch32/sys
-mount --rbind /dev /extra1/stretch32/dev
-mount --rbind /run/lock /extra1/stretch32/run/lock
+mount --rbind /tmp /opt/xenial32/tmp
+mount --rbind /proc /opt/xenial32/proc
+mount --rbind /sys /opt/xenial32/sys
+mount --rbind /dev /opt/xenial32/dev
+mount --rbind /run/lock /opt/xenial32/run/lock
 
 
 Commands to build the chroot
 
-mkdir /extra1/stretch32
+mkdir /opt/xenial32
 sudo debootstrap --variant=buildd --arch=i386 stretch \
-    /extra1/stretch32 http://deb.debian.org/debian
-LANG=C chroot /extra1/stretch32
-echo stretch32 > /etc/debian_chroot 
+    /opt/xenial32 http://archive.ubuntu.com/ubuntu/
+# or use arch=amd64
+LANG=C chroot /opt/xenial32
+echo xenial32 > /etc/debian_chroot 
 adduser buildslave
 # go back out out of the chroot and set buildslave's UID to the UID on the host
-chmod buildslave.buildslave -R /extra1/stretch32/home/buildslave
+# by editing /opt/xenial32/passwd, /opt/xenial32/group
+chown buildslave.buildslave -R /opt/xenial32/home/buildslave
 
+# now inside the chroot
 apt install virtualenv buildbot-slave python-pytest python-hypothesis \
 netbase gdb ncurses-term pypy locales mercurial
 
diff --git a/bot2/pypybuildbot/pypylist.py b/bot2/pypybuildbot/pypylist.py
--- a/bot2/pypybuildbot/pypylist.py
+++ b/bot2/pypybuildbot/pypylist.py
@@ -39,7 +39,7 @@
     PLATFORMS = {
         'linux':     'linux-x86-32',
         'linux64':   'linux-x86-64',
-        'aarch64':   'aarch64',
+        'aarch64':   'linux-aarch64',
         'osx':       'macosx-x86-32',
         'osx64':     'macosx-x86-64',
         'win32':     'win-x86-32',

From pypy.commits at gmail.com  Tue Oct 15 13:40:58 2019
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 15 Oct 2019 10:40:58 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6-asyncgen: fix translation
Message-ID: <5da604aa.1c69fb81.cce55.5e66@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6-asyncgen
Changeset: r97779:6bed9c5eebc5
Date: 2019-10-15 18:39 +0100
http://bitbucket.org/pypy/pypy/changeset/6bed9c5eebc5/

Log:	fix translation

diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -9,15 +9,15 @@
 # ____________________________________________________________
 
 
-_get_code = lambda frame, w_inputvalue, operr: frame.pycode
+_get_code = lambda frame, w_arg_or_err: frame.pycode
 _decorator = rvmprof.vmprof_execute_code("pypy", _get_code, W_Root)
 my_execute_frame = _decorator(PyFrame.execute_frame)
 
 
 class __extend__(PyFrame):
-    def execute_frame(self, in_generator=None, w_arg_or_err=None):
+    def execute_frame(self, w_arg_or_err=None):
         # indirection for the optional arguments
-        return my_execute_frame(self, in_generator, w_arg_or_err)
+        return my_execute_frame(self, w_arg_or_err)
 
 
 def _safe(s):

From pypy.commits at gmail.com  Tue Oct 15 13:43:30 2019
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 15 Oct 2019 10:43:30 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6-asyncgen: hg merge py3.6
Message-ID: <5da60542.1c69fb81.2086a.94ae@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6-asyncgen
Changeset: r97780:3ad9df8709f9
Date: 2019-10-15 18:42 +0100
http://bitbucket.org/pypy/pypy/changeset/3ad9df8709f9/

Log:	hg merge py3.6

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -50,4 +50,8 @@
 de061d87e39c7df4e436974096d7982c676a859d release-pypy3.6-v7.1.0
 784b254d669919c872a505b807db8462b6140973 release-pypy3.6-v7.1.1
 8cdda8b8cdb8ff29d9e620cccd6c5edd2f2a23ec release-pypy2.7-v7.1.1
-
+85dae4fd5c234b482feff834c73e089872194541 release-pypy2.7-v7.2.0rc0
+7ffb92269488f37c707ce66076f50ffd8613f8e2 release-pypy3.6-v7.2.0rc0
+4d6761df14ffd6f38450f183ac1fad32c946c21b release-pypy3.6-v7.2.0rc1
+5da45ced70e515f94686be0df47c59abd1348ebc release-pypy3.6-v7.2.0rc2
+4a68d8d3d2fc1faec2e83bcb4d28559099092574 release-pypy2.7-v7.2.0rc2
diff --git a/lib_pypy/_lzma.py b/lib_pypy/_lzma.py
--- a/lib_pypy/_lzma.py
+++ b/lib_pypy/_lzma.py
@@ -583,8 +583,16 @@
                 self.clear_input_buffer()
             elif lzs.avail_in == 0:
                 # completed successfully!
-                self.needs_input = True
                 lzs.next_in = ffi.NULL
+                if lzs.avail_out == 0:
+                    # (avail_in==0 && avail_out==0)
+                    # Maybe lzs's internal state still have a few bytes can
+                    # be output, try to output them next time.
+                    self.needs_input = False
+                    assert max_length >= 0   # if < 0, lzs.avail_out always > 0
+                else:
+                    # Input buffer exhausted, output buffer has space.
+                    self.needs_input = True
                 self.clear_input_buffer()
             else:
                 self.needs_input = False
@@ -599,9 +607,6 @@
         lzs.next_in = buf
         lzs.avail_in = buf_len
 
-        if buf_len == 0:
-            return b""
-
         bufsiz = self._bufsiz
         if not (max_length < 0 or max_length > io.DEFAULT_BUFFER_SIZE):
             bufsiz = max_length
@@ -616,7 +621,8 @@
 
         try:
             while True:
-                ret = catch_lzma_error(m.lzma_code, lzs, m.LZMA_RUN)
+                ret = catch_lzma_error(m.lzma_code, lzs, m.LZMA_RUN,
+                    ignore_buf_error=(lzs.avail_in == 0 and lzs.avail_out > 0))
                 data_size = int(ffi.cast('uintptr_t', lzs.next_out)) - int(ffi.cast('uintptr_t', orig_out))
                 # data_size is the amount lzma_code has already outputted
 
@@ -626,14 +632,17 @@
                 if ret == m.LZMA_STREAM_END:
                     self.eof = True
                     break
-                elif lzs.avail_in == 0:
-                    # it ate everything
-                    break
                 elif lzs.avail_out == 0:
+                    # Need to check lzs->avail_out before lzs->avail_in.
+                    # Maybe lzs's internal state still have a few bytes
+                    # can be output, grow the output buffer and continue
+                    # if max_lengh < 0.
                     if data_size == max_length:
                         break
                     # ran out of space in the output buffer, let's grow it
                     bufsiz += (bufsiz >> 3) + 6
+                    if max_length > 0 and bufsiz > max_length:
+                        bufsiz = max_length
                     next_out = m.realloc(orig_out, bufsiz)
                     if next_out == ffi.NULL:
                         # realloc unsuccessful
@@ -645,6 +654,9 @@
 
                     lzs.next_out = orig_out + data_size
                     lzs.avail_out = bufsiz - data_size
+                elif lzs.avail_in == 0:
+                    # it ate everything
+                    break
 
             result = ffi.buffer(orig_out, data_size)[:]
         finally:
diff --git a/pypy/doc/commandline_ref.rst b/pypy/doc/commandline_ref.rst
--- a/pypy/doc/commandline_ref.rst
+++ b/pypy/doc/commandline_ref.rst
@@ -9,3 +9,4 @@
 
    man/pypy.1.rst
    man/pypy3.1.rst
+   jit_help.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -42,7 +42,8 @@
 -------------------------------
 
 .. toctree::
-   whatsnew-pypy3-head.rst
+    whatsnew-pypy3-head.rst
+    whatsnew-pypy3-7.1.0.rst
 
 CPython 3.5 compatible versions
 -------------------------------
@@ -50,6 +51,8 @@
 .. toctree::
 
    whatsnew-pypy3-7.0.0.rst
+   whatsnew-pypy3-6.0.0.rst
+   whatsnew-pypy3-5.10.0.rst
    whatsnew-pypy3-5.9.0.rst
    whatsnew-pypy3-5.8.0.rst
    whatsnew-pypy3-5.7.0.rst
@@ -62,10 +65,4 @@
    whatsnew-pypy3-5.5.0.rst
    whatsnew-pypy3-5.1.1-alpha1.rst
 
-CPython 3.2 compatible versions
--------------------------------
 
-.. toctree::
-
-   whatsnew-pypy3-2.4.0.rst
-   whatsnew-pypy3-2.3.1.rst
diff --git a/pypy/doc/jit_help.rst b/pypy/doc/jit_help.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/jit_help.rst
@@ -0,0 +1,78 @@
+========
+JIT help
+========
+
+.. note this is from ``pypy --jit help``
+
+Advanced JIT options
+====================
+
+``<pypy> --jit`` [*options*] where *options* is a comma-separated list of
+``OPTION=VALUE``:
+
+ decay=N
+    amount to regularly decay counters by (0=none, 1000=max) (default 40)
+
+ disable_unrolling=N
+    after how many operations we should not unroll (default 200)
+
+ enable_opts=N
+    INTERNAL USE ONLY (MAY NOT WORK OR LEAD TO CRASHES): optimizations to
+    enable, or all =
+    intbounds:rewrite:virtualize:string:pure:earlyforce:heap:unroll (default
+    all)
+
+ function_threshold=N
+    number of times a function must run for it to become traced from start
+    (default 1619)
+
+ inlining=N
+    inline python functions or not (1/0) (default 1)
+
+ loop_longevity=N
+    a parameter controlling how long loops will be kept before being freed,
+    an estimate (default 1000)
+
+ max_retrace_guards=N
+    number of extra guards a retrace can cause (default 15)
+
+ max_unroll_loops=N
+    number of extra unrollings a loop can cause (default 0)
+
+ max_unroll_recursion=N
+    how many levels deep to unroll a recursive function (default 7)
+
+ retrace_limit=N
+    how many times we can try retracing before giving up (default 0)
+
+ threshold=N
+    number of times a loop has to run for it to become hot (default 1039)
+
+ trace_eagerness=N
+    number of times a guard has to fail before we start compiling a bridge
+    (default 200)
+
+ trace_limit=N
+    number of recorded operations before we abort tracing with ABORT_TOO_LONG
+    (default 6000)
+
+ vec=N
+    turn on the vectorization optimization (vecopt). Supports x86 (SSE 4.1),
+    powerpc (SVX), s390x SIMD (default 0)
+
+ vec_all=N
+    try to vectorize trace loops that occur outside of the numpypy library
+    (default 0)
+
+ vec_cost=N
+    threshold for which traces to bail. Unpacking increases the counter,
+    vector operation decrease the cost (default 0)
+
+ off
+    turn off the JIT
+ help
+    print this page
+
+The :ref:`pypyjit<jit-hooks>` module can be used to control the JIT from inside
+pypy
+
diff --git a/pypy/doc/release-v7.2.0.rst b/pypy/doc/release-v7.2.0.rst
--- a/pypy/doc/release-v7.2.0.rst
+++ b/pypy/doc/release-v7.2.0.rst
@@ -216,6 +216,7 @@
 * Add more constants to `sysconfig``. Set ``MACOSX_DEPLOYMENT_TARGET`` for
   darwin (`issue 2994`_)
 * fix ``CBuffer.buffer_attach``
+* Add ``_PyDict_GetItemWithError`` (``PyDict_GetItemWithError`` on Python3)
 
 Python 3.6 only
 ---------------
@@ -307,6 +308,7 @@
 .. _33786 : https://bugs.python.org/issue33786
 .. _32270 : https://bugs.python.org/issue32270
 .. _28691 : https://bugs.python.org/issue28691
+.. _33729 : https://bugs.python.org/issue33729
 
 .. _opencv2: https://github.com/skvark/opencv-python/
 .. _`issue 2617`: https://bitbucket.com/pypy/pypy/issues/2617
diff --git a/pypy/doc/whatsnew-pypy3-2.3.1.rst b/pypy/doc/whatsnew-pypy3-2.3.1.rst
deleted file mode 100644
--- a/pypy/doc/whatsnew-pypy3-2.3.1.rst
+++ /dev/null
@@ -1,6 +0,0 @@
-=========================
-What's new in PyPy3 2.3.1
-=========================
-
-.. this is a revision shortly after pypy3-release-2.3.x
-.. startrev: 0137d8e6657d
diff --git a/pypy/doc/whatsnew-pypy3-2.4.0.rst b/pypy/doc/whatsnew-pypy3-2.4.0.rst
deleted file mode 100644
--- a/pypy/doc/whatsnew-pypy3-2.4.0.rst
+++ /dev/null
@@ -1,6 +0,0 @@
-=========================
-What's new in PyPy3 2.4.0
-=========================
-
-.. this is a revision shortly after pypy3-release-2.4.x
-.. startrev: 12b940544622
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1790,8 +1790,7 @@
     def convert_arg_to_w_unicode(self, w_obj, strict=None):
         # XXX why convert_to_w_unicode does something slightly different?
         from pypy.objspace.std.unicodeobject import W_UnicodeObject
-        # for z_translation tests
-        if hasattr(self, 'is_fake_objspace'): return self.newtext("foobar")
+        assert not hasattr(self, 'is_fake_objspace')
         return W_UnicodeObject.convert_arg_to_w_unicode(self, w_obj, strict)
 
     def utf8_len_w(self, w_obj):
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -375,6 +375,14 @@
         return res_utf8, len(res), size
 
 def str_decode_utf8(s, errors, final, errorhandler, allow_surrogates=False):
+    try:
+        # fast version first
+        return s, rutf8.check_utf8(s, allow_surrogates=allow_surrogates), len(s)
+    except rutf8.CheckError:
+        return _str_decode_utf8_slowpath(
+            s, errors, final, errorhandler, allow_surrogates=allow_surrogates)
+
+def _str_decode_utf8_slowpath(s, errors, final, errorhandler, allow_surrogates):
     """ Same as checking for the valid utf8, but we know the utf8 is not
     valid so we're trying to either raise or pack stuff with error handler.
     The key difference is that this is call_may_force
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -737,17 +737,10 @@
         errors = 'strict'
     final = space.is_true(w_final)
     state = space.fromcache(CodecState)
-    # call the fast version for checking
-    try:
-        lgt = rutf8.check_utf8(string, allow_surrogates=False)
-    except rutf8.CheckError:
-        res, lgt, pos = unicodehelper.str_decode_utf8(string,
-            errors, final, state.decode_error_handler)
-        return space.newtuple([space.newutf8(res, lgt),
-                               space.newint(pos)])
-    else:
-        return space.newtuple([space.newutf8(string, lgt),
-                               space.newint(len(string))])
+    res, lgt, pos = unicodehelper.str_decode_utf8(string,
+        errors, final, state.decode_error_handler)
+    return space.newtuple([space.newutf8(res, lgt),
+                           space.newint(pos)])
 
 @unwrap_spec(data='bufferstr', errors='text_or_none', byteorder=int,
              w_final=WrappedDefault(False))
diff --git a/pypy/module/_csv/interp_csv.py b/pypy/module/_csv/interp_csv.py
--- a/pypy/module/_csv/interp_csv.py
+++ b/pypy/module/_csv/interp_csv.py
@@ -1,3 +1,4 @@
+from rpython.rlib import rutf8
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.typedef import TypeDef, interp_attrproperty
@@ -47,24 +48,26 @@
     if w_src is None:
         return default
     try:
-        return space.realunicode_w(w_src)
+        return space.text_w(w_src)
     except OperationError as e:
         if e.match(space, space.w_TypeError):
             raise oefmt(space.w_TypeError, '"%s" must be a string', attrname)
         raise
 
-def _get_char(space, w_src, default, name):
+def _get_codepoint(space, w_src, default, name):
     if w_src is None:
         return default
     if space.is_w(w_src, space.w_None):
-        return u'\0'
+        return 0
     if not space.isinstance_w(w_src, space.w_unicode):
         raise oefmt(space.w_TypeError, '"%s" must be string, not %T', name, w_src)
-    src = space.realunicode_w(w_src)
-    if len(src) == 1:
-        return src[0]
+    src, length = space.utf8_len_w(w_src)
+    if length == 1:
+        res = rutf8.codepoint_at_pos(src, 0)
+        assert res >= 0
+        return res
     if len(src) == 0:
-        return u'\0'
+        return 0
     raise oefmt(space.w_TypeError, '"%s" must be a 1-character string', name)
 
 def _build_dialect(space, w_dialect, w_delimiter, w_doublequote,
@@ -104,11 +107,11 @@
             w_strict = _fetch(space, w_dialect, 'strict')
 
     dialect = W_Dialect()
-    dialect.delimiter = _get_char(space, w_delimiter, u',', 'delimiter')
+    dialect.delimiter = _get_codepoint(space, w_delimiter, ord(u','), 'delimiter')
     dialect.doublequote = _get_bool(space, w_doublequote, True)
-    dialect.escapechar = _get_char(space, w_escapechar, u'\0', 'escapechar')
-    dialect.lineterminator = _get_str(space, w_lineterminator, u'\r\n', 'lineterminator')
-    dialect.quotechar = _get_char(space, w_quotechar, u'"', 'quotechar')
+    dialect.escapechar = _get_codepoint(space, w_escapechar, ord(u'\0'), 'escapechar')
+    dialect.lineterminator = _get_str(space, w_lineterminator, '\r\n', 'lineterminator')
+    dialect.quotechar = _get_codepoint(space, w_quotechar, ord(u'"'), 'quotechar')
     tmp_quoting = _get_int(space, w_quoting, QUOTE_MINIMAL, 'quoting')
     dialect.skipinitialspace = _get_bool(space, w_skipinitialspace, False)
     dialect.strict = _get_bool(space, w_strict, False)
@@ -117,13 +120,13 @@
     if not (0 <= tmp_quoting < 4):
         raise oefmt(space.w_TypeError, 'bad "quoting" value')
 
-    if dialect.delimiter == u'\0':
+    if dialect.delimiter == 0:
         raise oefmt(space.w_TypeError,
                     '"delimiter" must be a 1-character string')
 
     if space.is_w(w_quotechar, space.w_None) and w_quoting is None:
         tmp_quoting = QUOTE_NONE
-    if tmp_quoting != QUOTE_NONE and dialect.quotechar == u'\0':
+    if tmp_quoting != QUOTE_NONE and dialect.quotechar == 0:
         raise oefmt(space.w_TypeError,
                     "quotechar must be set if quoting enabled")
     dialect.quoting = tmp_quoting
@@ -158,14 +161,20 @@
 
 
 def _get_escapechar(space, dialect):
-    if dialect.escapechar == u'\0':
+    if dialect.escapechar == 0:
         return space.w_None
-    return space.newtext(dialect.escapechar)
+    s = rutf8.unichr_as_utf8(dialect.escapechar)
+    return space.newutf8(s, 1)
 
 def _get_quotechar(space, dialect):
-    if dialect.quotechar == u'\0':
+    if dialect.quotechar == 0:
         return space.w_None
-    return space.newtext(dialect.quotechar)
+    s = rutf8.unichr_as_utf8(dialect.quotechar)
+    return space.newutf8(s, 1)
+
+def _get_delimiter(space, dialect):
+    s = rutf8.unichr_as_utf8(dialect.delimiter)
+    return space.newutf8(s, 1)
 
 
 W_Dialect.typedef = TypeDef(
@@ -173,8 +182,7 @@
         __new__ = interp2app(W_Dialect___new__),
         __reduce_ex__ = interp2app(W_Dialect.reduce_ex_w),
 
-        delimiter        = interp_attrproperty('delimiter', W_Dialect,
-            wrapfn='newtext'),
+        delimiter        = GetSetProperty(_get_delimiter, cls=W_Dialect),
         doublequote      = interp_attrproperty('doublequote', W_Dialect,
             wrapfn='newbool'),
         escapechar       = GetSetProperty(_get_escapechar, cls=W_Dialect),
diff --git a/pypy/module/_csv/interp_reader.py b/pypy/module/_csv/interp_reader.py
--- a/pypy/module/_csv/interp_reader.py
+++ b/pypy/module/_csv/interp_reader.py
@@ -1,5 +1,4 @@
-from rpython.rlib.rstring import UnicodeBuilder
-from rpython.rlib.rutf8 import Utf8StringIterator
+from rpython.rlib.rutf8 import Utf8StringIterator, Utf8StringBuilder
 from rpython.rlib import objectmodel
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError
@@ -22,6 +21,7 @@
         self.dialect = dialect
         self.w_iter = w_iter
         self.line_num = 0
+        self.sizehint = 1  # just used for first line
 
     def iter_w(self):
         return self
@@ -38,22 +38,21 @@
         assert field_builder is not None
         if field_builder.getlength() >= field_limit.limit:
             raise self.error(u"field larger than field limit")
-        field_builder.append(c)
+        field_builder.append_code(c)
 
     def save_field(self, field_builder):
         space = self.space
         field = field_builder.build()
+        w_obj = space.newutf8(field, field_builder.getlength())
         if self.numeric_field:
             self.numeric_field = False
-            w_obj = space.call_function(space.w_float, space.newtext(field))
-        else:
-            w_obj = space.newtext(field)
+            w_obj = space.call_function(space.w_float, w_obj)
         self.fields_w.append(w_obj)
 
     def next_w(self):
         space = self.space
         dialect = self.dialect
-        self.fields_w = []
+        self.fields_w = objectmodel.newlist_hint(self.sizehint)
         self.numeric_field = False
         field_builder = None  # valid iff state not in [START_RECORD, EAT_CRNL]
         state = START_RECORD
@@ -79,13 +78,11 @@
                                  u"(did you open the file in text mode?")
             line = space.utf8_w(w_line)
             for c in Utf8StringIterator(line):
-                # XXX rewrite this to use c (as int) not unichr(c)
-                c = unichr(c)
-                if c == '\0':
+                if c == 0:
                     raise self.error(u"line contains NULL byte")
 
                 if state == START_RECORD:
-                    if c == b'\n' or c == b'\r':
+                    if c == ord(u'\n') or c == ord(u'\r'):
                         state = EAT_CRNL
                         continue
                     # normal character - handle as START_FIELD
@@ -93,9 +90,9 @@
                     # fall-through to the next case
 
                 if state == START_FIELD:
-                    field_builder = UnicodeBuilder(64)
+                    field_builder = Utf8StringBuilder(64)
                     # expecting field
-                    if c == u'\n' or c == u'\r':
+                    if c == ord(u'\n') or c == ord(u'\r'):
                         # save empty field
                         self.save_field(field_builder)
                         state = EAT_CRNL
@@ -106,7 +103,7 @@
                     elif c == dialect.escapechar:
                         # possible escaped character
                         state = ESCAPED_CHAR
-                    elif c == u' ' and dialect.skipinitialspace:
+                    elif c == ord(u' ') and dialect.skipinitialspace:
                         # ignore space at start of field
                         pass
                     elif c == dialect.delimiter:
@@ -120,7 +117,7 @@
                         state = IN_FIELD
 
                 elif state == ESCAPED_CHAR:
-                    if c in '\n\r':
+                    if c == ord(u'\n') or c == ord(u'\r'):
                         self.add_char(field_builder, c)
                         state = AFTER_ESCAPED_CRNL
                     else:
@@ -129,7 +126,7 @@
 
                 elif state == IN_FIELD or state == AFTER_ESCAPED_CRNL:
                     # in unquoted field
-                    if c == u'\n' or c == u'\r':
+                    if c == ord(u'\n') or c == ord(u'\r'):
                         # end of line
                         self.save_field(field_builder)
                         state = EAT_CRNL
@@ -176,7 +173,7 @@
                         # save field - wait for new field
                         self.save_field(field_builder)
                         state = START_FIELD
-                    elif c == u'\n' or c == u'\r':
+                    elif c == ord(u'\n') or c == ord(u'\r'):
                         # end of line
                         self.save_field(field_builder)
                         state = EAT_CRNL
@@ -186,10 +183,10 @@
                     else:
                         # illegal
                         raise self.error(u"'%s' expected after '%s'" % (
-                            dialect.delimiter, dialect.quotechar))
+                            unichr(dialect.delimiter), unichr(dialect.quotechar)))
 
                 elif state == EAT_CRNL:
-                    if not (c == u'\n' or c == u'\r'):
+                    if not (c == ord(u'\n') or c == ord(u'\r')):
                         raise self.error(u"new-line character seen in unquoted "
                                          u"field - do you need to open the file "
                                          u"in universal-newline mode?")
@@ -198,16 +195,16 @@
                 self.save_field(field_builder)
                 break
             elif state == ESCAPED_CHAR:
-                self.add_char(field_builder, u'\n')
+                self.add_char(field_builder, ord(u'\n'))
                 state = IN_FIELD
             elif state == IN_QUOTED_FIELD:
                 pass
             elif state == ESCAPE_IN_QUOTED_FIELD:
-                self.add_char(field_builder, u'\n')
+                self.add_char(field_builder, ord(u'\n'))
                 state = IN_QUOTED_FIELD
             elif state == START_FIELD:
                 # save empty field
-                field_builder = UnicodeBuilder(1)
+                field_builder = Utf8StringBuilder()
                 self.save_field(field_builder)
                 break
             elif state == AFTER_ESCAPED_CRNL:
@@ -216,6 +213,8 @@
                 break
         #
         w_result = space.newlist(self.fields_w)
+        # assume all lines have the same number of fields
+        self.sizehint = len(self.fields_w)
         self.fields_w = None
         return w_result
 
diff --git a/pypy/module/_csv/interp_writer.py b/pypy/module/_csv/interp_writer.py
--- a/pypy/module/_csv/interp_writer.py
+++ b/pypy/module/_csv/interp_writer.py
@@ -1,4 +1,4 @@
-from rpython.rlib.rstring import UnicodeBuilder
+from rpython.rlib.rutf8 import Utf8StringIterator, Utf8StringBuilder
 from rpython.rlib import objectmodel
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError
@@ -15,11 +15,13 @@
         self.dialect = dialect
         self.w_filewrite = space.getattr(w_fileobj, space.newtext('write'))
         # precompute this
-        special = dialect.delimiter + dialect.lineterminator
-        if dialect.escapechar != '\0':
-            special += dialect.escapechar
-        if dialect.quotechar != '\0':
-            special += dialect.quotechar
+        special = [dialect.delimiter]
+        for c in Utf8StringIterator(dialect.lineterminator):
+            special.append(c)
+        if dialect.escapechar != 0:
+            special.append(dialect.escapechar)
+        if dialect.quotechar != 0:
+            special.append(dialect.quotechar)
         self.special_characters = special
 
     @objectmodel.dont_inline
@@ -35,16 +37,17 @@
         space = self.space
         fields_w = space.listview(w_fields)
         dialect = self.dialect
-        rec = UnicodeBuilder(80)
+        rec = Utf8StringBuilder(80)
         #
         for field_index in range(len(fields_w)):
             w_field = fields_w[field_index]
             if space.is_w(w_field, space.w_None):
-                field = u""
+                field = ""
+                length = 0
             elif space.isinstance_w(w_field, space.w_float):
-                field = space.realunicode_w(space.repr(w_field))
+                field, length = space.utf8_len_w(space.repr(w_field))
             else:
-                field = space.realunicode_w(space.str(w_field))
+                field, length = space.utf8_len_w(space.str(w_field))
             #
             if dialect.quoting == QUOTE_NONNUMERIC:
                 try:
@@ -57,9 +60,9 @@
             elif dialect.quoting == QUOTE_ALL:
                 quoted = True
             elif dialect.quoting == QUOTE_MINIMAL:
-                # Find out if we really quoting
+                # Find out if we really need quoting.
                 special_characters = self.special_characters
-                for c in field:
+                for c in Utf8StringIterator(field):
                     if c in special_characters:
                         if c != dialect.quotechar or dialect.doublequote:
                             quoted = True
@@ -78,15 +81,15 @@
 
             # If this is not the first field we need a field separator
             if field_index > 0:
-                rec.append(dialect.delimiter)
+                rec.append_code(dialect.delimiter)
 
             # Handle preceding quote
             if quoted:
-                rec.append(dialect.quotechar)
+                rec.append_code(dialect.quotechar)
 
             # Copy field data
             special_characters = self.special_characters
-            for c in field:
+            for c in Utf8StringIterator(field):
                 if c in special_characters:
                     if dialect.quoting == QUOTE_NONE:
                         want_escape = True
@@ -94,28 +97,28 @@
                         want_escape = False
                         if c == dialect.quotechar:
                             if dialect.doublequote:
-                                rec.append(dialect.quotechar)
+                                rec.append_code(dialect.quotechar)
                             else:
                                 want_escape = True
                     if want_escape:
-                        if dialect.escapechar == u'\0':
+                        if dialect.escapechar == 0:
                             raise self.error("need to escape, "
                                              "but no escapechar set")
-                        rec.append(dialect.escapechar)
+                        rec.append_code(dialect.escapechar)
                     else:
                         assert quoted
                 # Copy field character into record buffer
-                rec.append(c)
+                rec.append_code(c)
 
             # Handle final quote
             if quoted:
-                rec.append(dialect.quotechar)
+                rec.append_code(dialect.quotechar)
 
         # Add line terminator
         rec.append(dialect.lineterminator)
 
         line = rec.build()
-        return space.call_function(self.w_filewrite, space.newtext(line))
+        return space.call_function(self.w_filewrite, space.newutf8(line, rec.getlength()))
 
     def writerows(self, w_seqseq):
         """Construct and write a series of sequences to a csv file.
diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -73,6 +73,9 @@
     # hit in the cache
     STRING_CACHE_USEFULNESS_FACTOR = 4
 
+    # don't make arbitrarily huge maps
+    MAX_MAP_SIZE = 100
+
 
     def __init__(self, space, s):
         self.space = space
@@ -369,7 +372,7 @@
                 return w_res
             elif ch == ',':
                 i = self.skip_whitespace(i)
-                if currmap.is_state_blocked():
+                if currmap.is_state_blocked() or nextindex > self.MAX_MAP_SIZE:
                     self.scratch.append(values_w)  # can reuse next time
                     dict_w = self._switch_to_dict(currmap, values_w, nextindex)
                     return self.decode_object_dict(i, start, dict_w)
diff --git a/pypy/module/_pypyjson/simd.py b/pypy/module/_pypyjson/simd.py
--- a/pypy/module/_pypyjson/simd.py
+++ b/pypy/module/_pypyjson/simd.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib import objectmodel, unroll
 from rpython.rlib.rarithmetic import r_uint, intmask, LONG_BIT
-from rpython.jit.backend.detect_cpu import autodetect
+from rpython.jit.backend.detect_cpu import autodetect, ProcessorAutodetectError
 
 # accelerators for string operations using simd on regular word sizes (*not*
 # SSE instructions). this style is sometimes called SWAR (SIMD Within A
@@ -15,8 +15,11 @@
     WORD_SIZE = 8
     EVERY_BYTE_ONE = 0x0101010101010101
     EVERY_BYTE_HIGHEST_BIT = 0x8080808080808080
-    if autodetect() == "x86-64":
-        USE_SIMD = True
+    try:
+        if autodetect() == "x86-64":
+            USE_SIMD = True
+    except ProcessorAutodetectError:
+        pass
 else:
     WORD_SIZE = 4
     EVERY_BYTE_ONE = 0x01010101
diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py
--- a/pypy/module/_pypyjson/test/test__pypyjson.py
+++ b/pypy/module/_pypyjson/test/test__pypyjson.py
@@ -467,6 +467,14 @@
         res = _pypyjson.loads(json)
         assert res == [{u'a': 1}, {u'a': 2}]
 
+    def test_huge_map(self):
+        import _pypyjson
+        import __pypy__
+        s = '{' + ",".join('"%s": %s' % (i, i) for i in range(200)) + '}'
+        res = _pypyjson.loads(s)
+        assert len(res) == 200
+        assert __pypy__.strategy(res) == "UnicodeDictStrategy"
+
     def test_tab_in_string_should_fail(self):
         import _pypyjson
         # http://json.org/JSON_checker/test/fail25.json
diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -80,6 +80,16 @@
     # XXX this is wrong with IntMutableCell.  Hope it works...
     return w_dict.getitem(w_key)
 
+ at cpython_api([PyObject, PyObject], PyObject, result_borrowed=True)
+def PyDict_GetItemWithError(space, w_dict, w_key):
+    """Variant of PyDict_GetItem() that does not suppress
+    exceptions. Return NULL with an exception set if an exception
+    occurred.  Return NULL without an exception set if the key
+    wasn't present."""
+    if not isinstance(w_dict, W_DictMultiObject):
+        PyErr_BadInternalCall(space)
+    return w_dict.getitem(w_key)
+
 @cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)
 def PyDict_SetItem(space, w_dict, w_key, w_obj):
     if not isinstance(w_dict, W_DictMultiObject):
@@ -298,3 +308,5 @@
             return 0
     return 1
 
+#def PyObject_GenericGetDict(space, w_obj, context):
+#    unlike CPython, you'll find this one in object.py together with ..SetDict
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -444,3 +444,16 @@
             del d[w_obj]
         except KeyError:
             pass
+
+ at cpython_api([PyObject, rffi.VOIDP], PyObject)
+def PyObject_GenericGetDict(space, w_obj, context):
+    from pypy.interpreter.typedef import descr_get_dict
+    return descr_get_dict(space, w_obj)
+
+ at cpython_api([PyObject, PyObject, rffi.VOIDP], rffi.INT_real, error=-1)
+def PyObject_GenericSetDict(space, w_obj, w_value, context):
+    from pypy.interpreter.typedef import descr_set_dict
+    if w_value is None:
+        raise oefmt(space.w_TypeError, "cannot delete __dict__")
+    descr_set_dict(space, w_obj, w_value)
+    return 0
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -243,14 +243,6 @@
 def PyWrapper_New(space, w_d, w_self):
     raise NotImplementedError
 
- at cpython_api([PyObject, PyObject], PyObject)
-def PyDict_GetItemWithError(space, p, key):
-    """Variant of PyDict_GetItem() that does not suppress
-    exceptions. Return NULL with an exception set if an exception
-    occurred.  Return NULL without an exception set if the key
-    wasn't present."""
-    raise NotImplementedError
-
 @cpython_api([PyObject, PyObject, rffi.INT_real], rffi.INT_real, error=-1)
 def PyDict_MergeFromSeq2(space, a, seq2, override):
     """Update or merge into dictionary a, from the key-value pairs in seq2.
diff --git a/pypy/module/cpyext/test/test_dictobject.py b/pypy/module/cpyext/test/test_dictobject.py
--- a/pypy/module/cpyext/test/test_dictobject.py
+++ b/pypy/module/cpyext/test/test_dictobject.py
@@ -175,6 +175,26 @@
             ])
         assert module.dict_proxy({'a': 1, 'b': 2}) == 2
 
+    def test_getitemwitherror(self):
+        module = self.import_extension('foo', [
+            ("dict_getitem", "METH_VARARGS",
+             """
+             PyObject *d, *key, *result;
+             if (!PyArg_ParseTuple(args, "OO", &d, &key)) {
+                return NULL;
+             }
+             result = PyDict_GetItemWithError(d, key);
+             if (result == NULL && !PyErr_Occurred())
+                Py_RETURN_NONE;
+             Py_XINCREF(result);
+             return result;
+             """)])
+        d = {'foo': 'bar'}
+        assert module.dict_getitem(d, 'foo') == 'bar'
+        assert module.dict_getitem(d, 'missing') is None
+        with raises(TypeError):
+            module.dict_getitem(d, [])
+
     def test_setdefault(self):
         module = self.import_extension('foo', [
             ("setdefault", "METH_VARARGS",
diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py
--- a/pypy/module/cpyext/test/test_object.py
+++ b/pypy/module/cpyext/test/test_object.py
@@ -451,6 +451,31 @@
         assert n == 1
         module.leave(obj2)
 
+    def test_GenericGetSetDict(self):
+        module = self.import_extension('test_GenericGetSetDict', [
+            ('test1', 'METH_VARARGS',
+             """
+                 PyObject *obj = PyTuple_GET_ITEM(args, 0);
+                 PyObject *newdict = PyTuple_GET_ITEM(args, 1);
+
+                 PyObject *olddict = PyObject_GenericGetDict(obj, NULL);
+                 if (olddict == NULL)
+                    return NULL;
+                 int res = PyObject_GenericSetDict(obj, newdict, NULL);
+                 if (res != 0)
+                     return NULL;
+                 return olddict;
+             """)])
+        class A:
+            pass
+        a = A()
+        a.x = 42
+        nd = {'y': 43}
+        d = module.test1(a, nd)
+        assert d == {'x': 42}
+        assert a.y == 43
+        assert a.__dict__ is nd
+
 
 class AppTestPyBuffer_FillInfo(AppTestCpythonExtensionBase):
     """
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -164,6 +164,9 @@
         is_root(w_obj)
         return NonConstant(False)
 
+    def utf8_len_w(self, space):
+        return NonConstant((NonConstant("utf8len_foobar"), NonConstant(14)))
+
     @not_rpython
     def unwrap(self, w_obj):
         raise NotImplementedError
diff --git a/pypy/objspace/fake/test/test_objspace.py b/pypy/objspace/fake/test/test_objspace.py
--- a/pypy/objspace/fake/test/test_objspace.py
+++ b/pypy/objspace/fake/test/test_objspace.py
@@ -1,4 +1,5 @@
-import py
+import pytest
+from rpython.rlib.nonconst import NonConstant
 from pypy.objspace.fake.objspace import FakeObjSpace, W_Root
 from pypy.interpreter.argument import Arguments
 from pypy.interpreter.typedef import TypeDef
@@ -63,8 +64,8 @@
     def test_is_true(self):
         space = self.space
         space.translates(lambda: space.is_true(W_Root()))
-        py.test.raises(AssertionError,
-                       space.translates, lambda: space.is_true(42))
+        with pytest.raises(AssertionError):
+            space.translates(lambda: space.is_true(42))
 
     def test_unpackiterable(self):
         space = self.space
@@ -79,3 +80,23 @@
         space = self.space
         space.translates(lambda: (space.get(W_Root(), W_Root()),
                                   space.get(W_Root(), W_Root(), W_Root())))
+
+    def test_bug_utf8_len_w(self):
+        space = self.space
+
+        class A(object):
+            pass
+
+        def f():
+            s = NonConstant('a')
+            w_s = space.newutf8(s, 1)
+            t, l = space.utf8_len_w(w_s)
+            a = A()
+            if l == 1:
+                a.x = 1
+            else:
+                raise Exception
+            return a.x
+        space.translates(f)
+
+
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -19,6 +19,7 @@
 
 class W_AbstractBytesObject(W_Root):
     __slots__ = ()
+    exact_class_applevel_name = 'bytes'
 
     def is_w(self, space, w_other):
         if not isinstance(w_other, W_AbstractBytesObject):
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -36,6 +36,10 @@
 def prev_codepoint_pos_dont_look_inside(utf8, p):
     return rutf8.prev_codepoint_pos(utf8, p)
 
+ at jit.elidable
+def codepoint_at_pos_dont_look_inside(utf8, p):
+    return rutf8.codepoint_at_pos(utf8, p)
+
 
 class W_UnicodeObject(W_Root):
     import_from_mixin(StringMethods)
@@ -122,7 +126,7 @@
             raise oefmt(space.w_TypeError,
                          "ord() expected a character, but string of length %d "
                          "found", self._len())
-        return space.newint(rutf8.codepoint_at_pos(self._utf8, 0))
+        return space.newint(self.codepoint_at_pos_dont_look_inside(0))
 
     def _empty(self):
         return W_UnicodeObject.EMPTY
@@ -548,7 +552,7 @@
         if self._length == 0:
             return space.w_False
         if self._length == 1:
-            return space.newbool(func(rutf8.codepoint_at_pos(self._utf8, 0)))
+            return space.newbool(func(self.codepoint_at_pos_dont_look_inside(0)))
         else:
             return self._is_generic_loop(space, self._utf8, func_name)
 
@@ -1127,6 +1131,11 @@
             return pos - 1
         return prev_codepoint_pos_dont_look_inside(self._utf8, pos)
 
+    def codepoint_at_pos_dont_look_inside(self, pos):
+        if self.is_ascii():
+            return ord(self._utf8[pos])
+        return codepoint_at_pos_dont_look_inside(self._utf8, pos)
+
     @always_inline
     def _unwrap_and_search(self, space, w_sub, w_start, w_end, forward=True):
         w_sub = self.convert_arg_to_w_unicode(space, w_sub)
diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,15 +1,18 @@
 # Edit these appropriately before running this script
 pmaj=2  # python main version: 2 or 3
 pmin=7  # python minor version
-exe=pypy3 # pypy3 or pypy
 maj=7
 min=2
-rev=0
+rev=0rc2
 
+case $pmaj in
+    "2") exe=pypy;;
+    "3") exe=pypy3;;
+    *) echo invalid pmaj=$pmaj; exit 1;;
+esac
 
 branchname=release-pypy$pmaj.$pmin-v$maj.x # ==OR== release-v$maj.x  # ==OR== release-v$maj.$min.x
-tagname=release-candidate-pypy$pmaj.$pmin-v$maj.$min.$rev  # ==OR== release-$maj.$min
-# tagname=release-pypy$pmaj.$pmin-v$maj.$min.$rev  # ==OR== release-$maj.$min
+tagname=release-pypy$pmaj.$pmin-v$maj.$min.$rev  # ==OR== release-$maj.$min
 
 echo checking hg log -r $branchname
 hg log -r $branchname || exit 1
diff --git a/requirements.txt b/requirements.txt
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,5 +6,5 @@
 vmprof>=0.4.10; 'x86' in platform.machine #skip arm, s390x
 
 # hypothesis is used for test generation on untranslated tests
-hypothesis
+hypothesis<4.40
 enum34>=1.1.2
diff --git a/rpython/jit/metainterp/quasiimmut.py b/rpython/jit/metainterp/quasiimmut.py
--- a/rpython/jit/metainterp/quasiimmut.py
+++ b/rpython/jit/metainterp/quasiimmut.py
@@ -5,6 +5,7 @@
 from rpython.jit.metainterp.history import (
     AbstractDescr, ConstPtr, ConstInt, ConstFloat)
 from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop
 
 
 def get_mutate_field_name(fieldname):
@@ -26,12 +27,14 @@
     return qmut
 
 def make_invalidation_function(STRUCT, mutatefieldname):
-    #
+    # fake a repr
+    descr_repr = "FieldDescr(%s, '%s')" % (STRUCT.TO, mutatefieldname)
+
     def _invalidate_now(p):
         qmut_ptr = getattr(p, mutatefieldname)
         setattr(p, mutatefieldname, lltype.nullptr(rclass.OBJECT))
         qmut = cast_base_ptr_to_instance(QuasiImmut, qmut_ptr)
-        qmut.invalidate()
+        qmut.invalidate(descr_repr)
     _invalidate_now._dont_inline_ = True
     #
     def invalidation(p):
@@ -45,7 +48,7 @@
     if qmut_ref:
         cpu.bh_setfield_gc_r(p, ConstPtr.value, mutatefielddescr)
         qmut = cast_gcref_to_instance(QuasiImmut, qmut_ref)
-        qmut.invalidate()
+        qmut.invalidate(mutatefielddescr.repr_of_descr())
 
 
 class QuasiImmut(object):
@@ -78,7 +81,8 @@
         # already invalidated; see below
         self.compress_limit = (len(self.looptokens_wrefs) + 15) * 2
 
-    def invalidate(self):
+    def invalidate(self, descr_repr=None):
+        debug_start("jit-invalidate-quasi-immutable")
         # When this is called, all the loops that we record become
         # invalid: all GUARD_NOT_INVALIDATED in these loops (and
         # in attached bridges) must now fail.
@@ -87,9 +91,11 @@
             return
         wrefs = self.looptokens_wrefs
         self.looptokens_wrefs = []
+        invalidated = 0
         for wref in wrefs:
             looptoken = wref()
             if looptoken is not None:
+                invalidated += 1
                 looptoken.invalidated = True
                 self.cpu.invalidate_loop(looptoken)
                 # NB. we must call cpu.invalidate_loop() even if
@@ -100,6 +106,8 @@
                 if not we_are_translated():
                     self.cpu.stats.invalidated_token_numbers.add(
                         looptoken.number)
+        debug_print("fieldname", descr_repr or "<unknown>", "invalidated", invalidated)
+        debug_stop("jit-invalidate-quasi-immutable")
 
 
 class QuasiImmutDescr(AbstractDescr):
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -1903,7 +1903,9 @@
                 if MultiByteToWideChar(CP_ACP, flags,
                                        dataptr, size, buf.raw, usize) == 0:
                     _decode_mbcs_error(s, errorhandler)
-                return buf.str(usize), size
+                ret = buf.str(usize)
+                assert ret is not None
+                return ret, size
 
     def unicode_encode_mbcs(s, size, errors, errorhandler=None,
                             force_replace=True):
diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
--- a/rpython/translator/revdb/src-revdb/revdb_include.h
+++ b/rpython/translator/revdb/src-revdb/revdb_include.h
@@ -285,6 +285,8 @@
 #define OP_GC_RAWREFCOUNT_NEXT_DEAD(r)   \
     r = rpy_reverse_db_rawrefcount_next_dead()
 
+#define OP_GC_INCREASE_ROOT_STACK_DEPTH(depth, r)   /* nothing */
+
 
 RPY_EXTERN void rpy_reverse_db_flush(void);  /* must be called with the lock */
 RPY_EXTERN void rpy_reverse_db_fetch(const char *file, int line);

From pypy.commits at gmail.com  Wed Oct 16 02:00:23 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 15 Oct 2019 23:00:23 -0700 (PDT)
Subject: [pypy-commit] pypy fix-descrmismatch-crash: close branch to be
 merged
Message-ID: <5da6b1f7.1c69fb81.1f155.7ea5@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: fix-descrmismatch-crash
Changeset: r97782:4b90fb018acd
Date: 2019-10-16 08:13 +0300
http://bitbucket.org/pypy/pypy/changeset/4b90fb018acd/

Log:	close branch to be merged

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -3,11 +3,9 @@
 ==========================
 
 .. this is a revision shortly after release-pypy-7.2.0
-.. startrev: 78cd4acbcbec 
+.. startrev: a511d86377d6 
 
+.. branch: fix-descrmismatch-crash
 
-.. branch: json-decoder-maps
+Fix segfault when calling descr-methods with no arguments
 
-Much faster and more memory-efficient JSON decoding. The resulting
-dictionaries that come out of the JSON decoder have faster lookups too.
-
diff --git a/pypy/doc/whatsnew-pypy2-7.2.0.rst b/pypy/doc/whatsnew-pypy2-7.2.0.rst
--- a/pypy/doc/whatsnew-pypy2-7.2.0.rst
+++ b/pypy/doc/whatsnew-pypy2-7.2.0.rst
@@ -74,3 +74,9 @@
 .. branch: openssl-for-macos
 
 Update _ssl on macos to statically link to openssl-1.1.1c
+
+.. branch: json-decoder-maps
+
+Much faster and more memory-efficient JSON decoding. The resulting
+dictionaries that come out of the JSON decoder have faster lookups too.
+

From pypy.commits at gmail.com  Wed Oct 16 02:00:25 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 15 Oct 2019 23:00:25 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge branch to fix descrmismatch
 exception
Message-ID: <5da6b1f9.1c69fb81.f8e83.62bb@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r97783:f089d461d2ee
Date: 2019-10-16 08:14 +0300
http://bitbucket.org/pypy/pypy/changeset/f089d461d2ee/

Log:	merge branch to fix descrmismatch exception

diff --git a/lib-python/2.7/test/test_dictviews.py b/lib-python/2.7/test/test_dictviews.py
--- a/lib-python/2.7/test/test_dictviews.py
+++ b/lib-python/2.7/test/test_dictviews.py
@@ -182,7 +182,7 @@
 
     def test_deeply_nested_repr(self):
         d = {}
-        for i in range(sys.getrecursionlimit() + 100):
+        for i in range(sys.getrecursionlimit() + 200):
             d = {42: d.viewvalues()}
         self.assertRaises(RuntimeError, repr, d)
 
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -3,11 +3,9 @@
 ==========================
 
 .. this is a revision shortly after release-pypy-7.2.0
-.. startrev: 78cd4acbcbec 
+.. startrev: a511d86377d6 
 
+.. branch: fix-descrmismatch-crash
 
-.. branch: json-decoder-maps
+Fix segfault when calling descr-methods with no arguments
 
-Much faster and more memory-efficient JSON decoding. The resulting
-dictionaries that come out of the JSON decoder have faster lookups too.
-
diff --git a/pypy/doc/whatsnew-pypy2-7.2.0.rst b/pypy/doc/whatsnew-pypy2-7.2.0.rst
--- a/pypy/doc/whatsnew-pypy2-7.2.0.rst
+++ b/pypy/doc/whatsnew-pypy2-7.2.0.rst
@@ -74,3 +74,9 @@
 .. branch: openssl-for-macos
 
 Update _ssl on macos to statically link to openssl-1.1.1c
+
+.. branch: json-decoder-maps
+
+Much faster and more memory-efficient JSON decoding. The resulting
+dictionaries that come out of the JSON decoder have faster lookups too.
+
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -419,6 +419,8 @@
 
 @specialize.memo()
 def wrappable_class_name(Class):
+    if 'exact_class_applevel_name' in Class.__dict__:
+        return Class.exact_class_applevel_name
     try:
         return Class.typedef.name
     except AttributeError:
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -709,6 +709,7 @@
                     self.func__args__ = func
                 elif unwrap_spec == [self_type, ObjSpace, Arguments]:
                     self.__class__ = BuiltinCodePassThroughArguments1
+                    self.descr_reqcls = self_type
                     miniglobals = {'func': func, 'self_type': self_type}
                     d = {}
                     source = """if 1:
@@ -754,10 +755,7 @@
         except DescrMismatch:
             if w_obj is not None:
                 args = args.prepend(w_obj)
-            return scope_w[0].descr_call_mismatch(space,
-                                                  self.descrmismatch_op,
-                                                  self.descr_reqcls,
-                                                  args)
+            return self._type_unwrap_mismatch(space, args)
         except Exception as e:
             self.handle_exception(space, e)
             w_result = None
@@ -765,6 +763,15 @@
             w_result = space.w_None
         return w_result
 
+    def _type_unwrap_mismatch(self, space, args):
+        w_obj = args.firstarg()
+        if w_obj is None:
+            raise oefmt(space.w_SystemError, "unexpected DescrMismatch error")
+        return w_obj.descr_call_mismatch(space,
+                                         self.descrmismatch_op,
+                                         self.descr_reqcls,
+                                         args)
+
     def handle_exception(self, space, e):
         try:
             if not we_are_translated():
@@ -787,10 +794,7 @@
         try:
             w_result = self.func__args__(space, args)
         except DescrMismatch:
-            return args.firstarg().descr_call_mismatch(space,
-                                                  self.descrmismatch_op,
-                                                  self.descr_reqcls,
-                                                  args)
+            return self._type_unwrap_mismatch(space, args)
         except Exception as e:
             self.handle_exception(space, e)
             w_result = None
@@ -808,10 +812,7 @@
         try:
             w_result = self.func__args__(space, w_obj, args)
         except DescrMismatch:
-            return args.firstarg().descr_call_mismatch(space,
-                                                  self.descrmismatch_op,
-                                                  self.descr_reqcls,
-                                                  args.prepend(w_obj))
+            return self._type_unwrap_mismatch(space, args.prepend(w_obj))
         except Exception as e:
             self.handle_exception(space, e)
             w_result = None
@@ -851,9 +852,7 @@
         try:
             w_result = self.fastfunc_1(space, w1)
         except DescrMismatch:
-            return w1.descr_call_mismatch(space,
-                                          self.descrmismatch_op,
-                                          self.descr_reqcls,
+            return self._type_unwrap_mismatch(space,
                                           Arguments(space, [w1]))
         except Exception as e:
             self.handle_exception(space, e)
@@ -877,9 +876,7 @@
         try:
             w_result = self.fastfunc_2(space, w1, w2)
         except DescrMismatch:
-            return w1.descr_call_mismatch(space,
-                                          self.descrmismatch_op,
-                                          self.descr_reqcls,
+            return self._type_unwrap_mismatch(space,
                                           Arguments(space, [w1, w2]))
         except Exception as e:
             self.handle_exception(space, e)
@@ -904,9 +901,7 @@
         try:
             w_result = self.fastfunc_3(space, w1, w2, w3)
         except DescrMismatch:
-            return w1.descr_call_mismatch(space,
-                                          self.descrmismatch_op,
-                                          self.descr_reqcls,
+            return self._type_unwrap_mismatch(space,
                                           Arguments(space, [w1, w2, w3]))
         except Exception as e:
             self.handle_exception(space, e)
@@ -932,9 +927,7 @@
         try:
             w_result = self.fastfunc_4(space, w1, w2, w3, w4)
         except DescrMismatch:
-            return w1.descr_call_mismatch(space,
-                                          self.descrmismatch_op,
-                                          self.descr_reqcls,
+            return self._type_unwrap_mismatch(space,
                                           Arguments(space,
                                                     [w1, w2, w3, w4]))
         except Exception as e:
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -966,6 +966,29 @@
         # white-box check for opt
         assert called[0] is args
 
+    def test_base_regular_descr_mismatch(self):
+        space = self.space
+
+        def f():
+            raise gateway.DescrMismatch
+
+        w_f = space.wrap(gateway.interp2app_temp(f,
+                         unwrap_spec=[]))
+        args = argument.Arguments(space, [])
+        space.raises_w(space.w_SystemError, space.call_args, w_f, args)
+
+    def test_pass_trough_arguments0_descr_mismatch(self):
+        space = self.space
+
+        def f(space, __args__):
+            raise gateway.DescrMismatch
+
+        w_f = space.wrap(gateway.interp2app_temp(f,
+                         unwrap_spec=[gateway.ObjSpace,
+                                      gateway.Arguments]))
+        args = argument.Arguments(space, [])
+        space.raises_w(space.w_SystemError, space.call_args, w_f, args)
+
 
 class AppTestKeywordsToBuiltinSanity(object):
     def test_type(self):
@@ -1005,3 +1028,26 @@
 
         d.update(**{clash: 33})
         dict.update(d, **{clash: 33})
+
+
+
+class AppTestFastPathCrash(object):
+    def setup_class(cls):
+        cls.w_runappdirect = cls.space.wrap(cls.runappdirect)
+
+    def test_fast_path_crash(self):
+        # issue bb-3091 crash in BuiltinCodePassThroughArguments0.funcrun
+        for obj in (dict, set):
+            with raises(TypeError) as excinfo:
+                if self.runappdirect:
+                    import platform
+                    if platform.python_implementation() == 'PyPy':
+                        msg_fmt = "%s instance as first argument (got %s"
+                    else:
+                        msg_fmt = "'%s' object but received a '%s'"
+                    obj.__init__(0)
+                else:
+                    msg_fmt = "'%s' object expected, got '%s'"
+                    obj.__init__.im_func(0)
+            msg = msg_fmt %(obj.__name__, 'int')
+            assert msg in str(excinfo.value)
diff --git a/pypy/module/_weakref/interp__weakref.py b/pypy/module/_weakref/interp__weakref.py
--- a/pypy/module/_weakref/interp__weakref.py
+++ b/pypy/module/_weakref/interp__weakref.py
@@ -156,6 +156,8 @@
 
 
 class W_WeakrefBase(W_Root):
+    exact_class_applevel_name = 'weakref-or-proxy'
+
     def __init__(self, space, w_obj, w_callable):
         assert w_callable is not space.w_None    # should be really None
         self.space = space
diff --git a/pypy/module/_weakref/test/test_weakref.py b/pypy/module/_weakref/test/test_weakref.py
--- a/pypy/module/_weakref/test/test_weakref.py
+++ b/pypy/module/_weakref/test/test_weakref.py
@@ -543,3 +543,12 @@
         p1[42] = p2
         assert a1.setkey == 42
         assert a1.setvalue is p2
+
+    def test_error_message_wrong_self(self):
+        import _weakref
+        unboundmeth = _weakref.ref.__repr__
+        e = raises(TypeError, unboundmeth, 42)
+        assert "weakref" in str(e.value)
+        if hasattr(unboundmeth, 'im_func'):
+            e = raises(TypeError, unboundmeth.im_func, 42)
+            assert "'weakref-or-proxy'" in str(e.value)
diff --git a/pypy/module/cStringIO/interp_stringio.py b/pypy/module/cStringIO/interp_stringio.py
--- a/pypy/module/cStringIO/interp_stringio.py
+++ b/pypy/module/cStringIO/interp_stringio.py
@@ -7,6 +7,8 @@
 
 
 class W_InputOutputType(W_Root):
+    exact_class_applevel_name = "StringI-or-StringO"
+
     softspace = 0    # part of the file object API
 
     def descr___iter__(self):
diff --git a/pypy/module/cStringIO/test/test_interp_stringio.py b/pypy/module/cStringIO/test/test_interp_stringio.py
--- a/pypy/module/cStringIO/test/test_interp_stringio.py
+++ b/pypy/module/cStringIO/test/test_interp_stringio.py
@@ -204,3 +204,12 @@
         import cStringIO
         assert type(cStringIO.StringIO()) is cStringIO.OutputType
         assert type(cStringIO.StringIO('')) is cStringIO.InputType
+
+    def test_error_message_wrong_self(self):
+        import cStringIO
+        unboundmeth = cStringIO.InputType.close
+        e = raises(TypeError, unboundmeth, 42)
+        assert "StringI" in str(e.value)
+        if hasattr(unboundmeth, 'im_func'):
+            e = raises(TypeError, unboundmeth.im_func, 42)
+            assert "'StringI-or-StringO'" in str(e.value)
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -24,6 +24,7 @@
 
 class W_AbstractBytesObject(W_Root):
     __slots__ = ()
+    exact_class_applevel_name = 'str'
 
     def is_w(self, space, w_other):
         if not isinstance(w_other, W_AbstractBytesObject):
diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -20,6 +20,7 @@
 
 class W_BaseSetObject(W_Root):
     typedef = None
+    exact_class_applevel_name = 'set-or-frozenset'
 
     def __init__(self, space, w_iterable=None):
         """Initialize the set by taking ownership of 'setdata'."""
@@ -496,6 +497,12 @@
 
 
 class W_SetObject(W_BaseSetObject):
+
+    #overridden here so the error is reported correctly
+    def __init__(self, space, w_iterable=None):
+        """Initialize the set by taking ownership of 'setdata'."""
+        W_BaseSetObject.__init__(self, space, w_iterable)
+
     def _newobj(self, space, w_iterable):
         """Make a new set by taking ownership of 'w_iterable'."""
         if type(self) is W_SetObject:
@@ -516,7 +523,7 @@
 
 Build an unordered collection.""",
     __new__ = gateway.interp2app(W_SetObject.descr_new),
-    __init__ = gateway.interp2app(W_BaseSetObject.descr_init),
+    __init__ = gateway.interp2app(W_SetObject.descr_init),
     __repr__ = gateway.interp2app(W_BaseSetObject.descr_repr),
     __hash__ = None,
     __cmp__ = gateway.interp2app(W_BaseSetObject.descr_cmp),
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -933,3 +933,10 @@
     def test_add(self):
         assert 'abc' + 'abc' == 'abcabc'
         assert isinstance('abc' + u'\u03a3', unicode)
+
+    def test_error_message_wrong_self(self):
+        e = raises(TypeError, bytes.upper, 42)
+        assert "str" in str(e.value)
+        if hasattr(bytes.upper, 'im_func'):
+            e = raises(TypeError, bytes.upper.im_func, 42)
+            assert "'str'" in str(e.value)
diff --git a/pypy/objspace/std/test/test_intobject.py b/pypy/objspace/std/test/test_intobject.py
--- a/pypy/objspace/std/test/test_intobject.py
+++ b/pypy/objspace/std/test/test_intobject.py
@@ -684,6 +684,15 @@
         x = int(-sys.maxint)
         assert x.__rsub__(2) == (2 + sys.maxint)
 
+    def test_error_message_wrong_self(self):
+        unboundmeth = int.__str__
+        e = raises(TypeError, unboundmeth, "!")
+        assert "int" in str(e.value)
+        if hasattr(unboundmeth, 'im_func'):
+            e = raises(TypeError, unboundmeth.im_func, "!")
+            assert "'int'" in str(e.value)
+
+
 class AppTestIntShortcut(AppTestInt):
     spaceconfig = {"objspace.std.intshortcut": True}
 
diff --git a/pypy/objspace/std/test/test_longobject.py b/pypy/objspace/std/test/test_longobject.py
--- a/pypy/objspace/std/test/test_longobject.py
+++ b/pypy/objspace/std/test/test_longobject.py
@@ -450,3 +450,10 @@
         expected = (2 << (size * 4)) // 3
         assert long(n, 16) == expected
 
+    def test_error_message_wrong_self(self):
+        unboundmeth = long.__str__
+        e = raises(TypeError, unboundmeth, 42)
+        assert "long" in str(e.value)
+        if hasattr(unboundmeth, 'im_func'):
+            e = raises(TypeError, unboundmeth.im_func, 42)
+            assert "'long'" in str(e.value)
diff --git a/pypy/objspace/std/test/test_setobject.py b/pypy/objspace/std/test/test_setobject.py
--- a/pypy/objspace/std/test/test_setobject.py
+++ b/pypy/objspace/std/test/test_setobject.py
@@ -1037,3 +1037,15 @@
            raise ValueError
            yield 1
         raises(ValueError, set, f())
+
+    def test_frozenset_init_does_nothing(self):
+        f = frozenset([1, 2, 3])
+        f.__init__(4, 5, 6)
+        assert f == frozenset([1, 2, 3])
+
+    def test_error_message_wrong_self(self):
+        e = raises(TypeError, frozenset.copy, 42)
+        assert "frozenset" in str(e.value)
+        if hasattr(frozenset.copy, 'im_func'):
+            e = raises(TypeError, frozenset.copy.im_func, 42)
+            assert "'set-or-frozenset'" in str(e.value)
diff --git a/pypy/objspace/std/test/test_tupleobject.py b/pypy/objspace/std/test/test_tupleobject.py
--- a/pypy/objspace/std/test/test_tupleobject.py
+++ b/pypy/objspace/std/test/test_tupleobject.py
@@ -454,3 +454,11 @@
             (4.1, 2.3), (3.6, 4.8)]
         assert specialized_zip_2_lists(["foo", "bar"], [6, 2]) == [
             ("foo", 6), ("bar", 2)]
+
+    def test_error_message_wrong_self(self):
+        unboundmeth = tuple.__hash__
+        e = raises(TypeError, unboundmeth, 42)
+        assert "tuple" in str(e.value)
+        if hasattr(unboundmeth, 'im_func'):
+            e = raises(TypeError, unboundmeth.im_func, 42)
+            assert "'tuple'" in str(e.value)

From pypy.commits at gmail.com  Wed Oct 16 02:00:27 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 15 Oct 2019 23:00:27 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: merge default into branch
Message-ID: <5da6b1fb.1c69fb81.c2a35.715e@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r97784:108d3c00537f
Date: 2019-10-16 08:22 +0300
http://bitbucket.org/pypy/pypy/changeset/108d3c00537f/

Log:	merge default into branch

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -55,3 +55,5 @@
 4d6761df14ffd6f38450f183ac1fad32c946c21b release-pypy3.6-v7.2.0rc1
 5da45ced70e515f94686be0df47c59abd1348ebc release-pypy3.6-v7.2.0rc2
 4a68d8d3d2fc1faec2e83bcb4d28559099092574 release-pypy2.7-v7.2.0rc2
+4a68d8d3d2fc1faec2e83bcb4d28559099092574 release-pypy2.7-v7.2.0
+5da45ced70e515f94686be0df47c59abd1348ebc release-pypy3.6-v7.2.0
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -3,11 +3,9 @@
 ==========================
 
 .. this is a revision shortly after release-pypy-7.2.0
-.. startrev: 78cd4acbcbec 
+.. startrev: a511d86377d6 
 
+.. branch: fix-descrmismatch-crash
 
-.. branch: json-decoder-maps
+Fix segfault when calling descr-methods with no arguments
 
-Much faster and more memory-efficient JSON decoding. The resulting
-dictionaries that come out of the JSON decoder have faster lookups too.
-
diff --git a/pypy/doc/whatsnew-pypy2-7.2.0.rst b/pypy/doc/whatsnew-pypy2-7.2.0.rst
--- a/pypy/doc/whatsnew-pypy2-7.2.0.rst
+++ b/pypy/doc/whatsnew-pypy2-7.2.0.rst
@@ -17,10 +17,6 @@
 
 Add ``DateTime_FromTimestamp`` and ``Date_FromTimestamp``
 
-.. branch: issue2968
-
-Fix segfault in cpyext_tp_new_tupl
-
 .. branch: semlock-deadlock
 
 Test and reduce the probability of a deadlock when acquiring a semaphore by
@@ -79,6 +75,8 @@
 
 Update _ssl on macos to statically link to openssl-1.1.1c
 
-.. branch: more-cpyext
+.. branch: json-decoder-maps
 
-Add more datetime C functions and definitions
+Much faster and more memory-efficient JSON decoding. The resulting
+dictionaries that come out of the JSON decoder have faster lookups too.
+
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -394,6 +394,8 @@
 
 @specialize.memo()
 def wrappable_class_name(Class):
+    if 'exact_class_applevel_name' in Class.__dict__:
+        return Class.exact_class_applevel_name
     try:
         return Class.typedef.name
     except AttributeError:
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -755,6 +755,7 @@
                     self.func__args__ = func
                 elif unwrap_spec == [self_type, ObjSpace, Arguments]:
                     self.__class__ = BuiltinCodePassThroughArguments1
+                    self.descr_reqcls = self_type
                     miniglobals = {'func': func, 'self_type': self_type}
                     d = {}
                     source = """if 1:
@@ -800,10 +801,7 @@
         except DescrMismatch:
             if w_obj is not None:
                 args = args.prepend(w_obj)
-            return scope_w[0].descr_call_mismatch(space,
-                                                  self.descrmismatch_op,
-                                                  self.descr_reqcls,
-                                                  args)
+            return self._type_unwrap_mismatch(space, args)
         except Exception as e:
             self.handle_exception(space, e)
             w_result = None
@@ -811,6 +809,15 @@
             w_result = space.w_None
         return w_result
 
+    def _type_unwrap_mismatch(self, space, args):
+        w_obj = args.firstarg()
+        if w_obj is None:
+            raise oefmt(space.w_SystemError, "unexpected DescrMismatch error")
+        return w_obj.descr_call_mismatch(space,
+                                         self.descrmismatch_op,
+                                         self.descr_reqcls,
+                                         args)
+
     def handle_exception(self, space, e):
         try:
             if not we_are_translated():
@@ -833,10 +840,7 @@
         try:
             w_result = self.func__args__(space, args)
         except DescrMismatch:
-            return args.firstarg().descr_call_mismatch(space,
-                                                  self.descrmismatch_op,
-                                                  self.descr_reqcls,
-                                                  args)
+            return self._type_unwrap_mismatch(space, args)
         except Exception as e:
             self.handle_exception(space, e)
             w_result = None
@@ -854,10 +858,7 @@
         try:
             w_result = self.func__args__(space, w_obj, args)
         except DescrMismatch:
-            return args.firstarg().descr_call_mismatch(space,
-                                                  self.descrmismatch_op,
-                                                  self.descr_reqcls,
-                                                  args.prepend(w_obj))
+            return self._type_unwrap_mismatch(space, args.prepend(w_obj))
         except Exception as e:
             self.handle_exception(space, e)
             w_result = None
@@ -897,9 +898,7 @@
         try:
             w_result = self.fastfunc_1(space, w1)
         except DescrMismatch:
-            return w1.descr_call_mismatch(space,
-                                          self.descrmismatch_op,
-                                          self.descr_reqcls,
+            return self._type_unwrap_mismatch(space,
                                           Arguments(space, [w1]))
         except Exception as e:
             self.handle_exception(space, e)
@@ -923,9 +922,7 @@
         try:
             w_result = self.fastfunc_2(space, w1, w2)
         except DescrMismatch:
-            return w1.descr_call_mismatch(space,
-                                          self.descrmismatch_op,
-                                          self.descr_reqcls,
+            return self._type_unwrap_mismatch(space,
                                           Arguments(space, [w1, w2]))
         except Exception as e:
             self.handle_exception(space, e)
@@ -950,9 +947,7 @@
         try:
             w_result = self.fastfunc_3(space, w1, w2, w3)
         except DescrMismatch:
-            return w1.descr_call_mismatch(space,
-                                          self.descrmismatch_op,
-                                          self.descr_reqcls,
+            return self._type_unwrap_mismatch(space,
                                           Arguments(space, [w1, w2, w3]))
         except Exception as e:
             self.handle_exception(space, e)
@@ -978,9 +973,7 @@
         try:
             w_result = self.fastfunc_4(space, w1, w2, w3, w4)
         except DescrMismatch:
-            return w1.descr_call_mismatch(space,
-                                          self.descrmismatch_op,
-                                          self.descr_reqcls,
+            return self._type_unwrap_mismatch(space,
                                           Arguments(space,
                                                     [w1, w2, w3, w4]))
         except Exception as e:
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -1095,6 +1095,29 @@
         # white-box check for opt
         assert called[0] is args
 
+    def test_base_regular_descr_mismatch(self):
+        space = self.space
+
+        def f():
+            raise gateway.DescrMismatch
+
+        w_f = space.wrap(gateway.interp2app_temp(f,
+                         unwrap_spec=[]))
+        args = argument.Arguments(space, [])
+        space.raises_w(space.w_SystemError, space.call_args, w_f, args)
+
+    def test_pass_trough_arguments0_descr_mismatch(self):
+        space = self.space
+
+        def f(space, __args__):
+            raise gateway.DescrMismatch
+
+        w_f = space.wrap(gateway.interp2app_temp(f,
+                         unwrap_spec=[gateway.ObjSpace,
+                                      gateway.Arguments]))
+        args = argument.Arguments(space, [])
+        space.raises_w(space.w_SystemError, space.call_args, w_f, args)
+
 
 class AppTestKeywordsToBuiltinSanity(object):
     def test_type(self):
@@ -1134,3 +1157,26 @@
 
         d.update(**{clash: 33})
         dict.update(d, **{clash: 33})
+
+
+
+class AppTestFastPathCrash(object):
+    def setup_class(cls):
+        cls.w_runappdirect = cls.space.wrap(cls.runappdirect)
+
+    def test_fast_path_crash(self):
+        # issue bb-3091 crash in BuiltinCodePassThroughArguments0.funcrun
+        for obj in (dict, set):
+            with raises(TypeError) as excinfo:
+                if self.runappdirect:
+                    import platform
+                    if platform.python_implementation() == 'PyPy':
+                        msg_fmt = "%s instance as first argument (got %s"
+                    else:
+                        msg_fmt = "'%s' object but received a '%s'"
+                    obj.__init__(0)
+                else:
+                    msg_fmt = "'%s' object expected, got '%s'"
+                    obj.__init__.im_func(0)
+            msg = msg_fmt %(obj.__name__, 'int')
+            assert msg in str(excinfo.value)
diff --git a/pypy/module/_weakref/interp__weakref.py b/pypy/module/_weakref/interp__weakref.py
--- a/pypy/module/_weakref/interp__weakref.py
+++ b/pypy/module/_weakref/interp__weakref.py
@@ -157,6 +157,8 @@
 
 
 class W_WeakrefBase(W_Root):
+    exact_class_applevel_name = 'weakref-or-proxy'
+
     def __init__(self, space, w_obj, w_callable):
         assert w_callable is not space.w_None    # should be really None
         self.space = space
diff --git a/pypy/module/_weakref/test/test_weakref.py b/pypy/module/_weakref/test/test_weakref.py
--- a/pypy/module/_weakref/test/test_weakref.py
+++ b/pypy/module/_weakref/test/test_weakref.py
@@ -545,3 +545,12 @@
         p1[42] = p2
         assert a1.setkey == 42
         assert a1.setvalue is p2
+
+    def test_error_message_wrong_self(self):
+        import _weakref
+        unboundmeth = _weakref.ref.__repr__
+        e = raises(TypeError, unboundmeth, 42)
+        assert "weakref" in str(e.value)
+        if hasattr(unboundmeth, 'im_func'):
+            e = raises(TypeError, unboundmeth.im_func, 42)
+            assert "'weakref-or-proxy'" in str(e.value)
diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -20,6 +20,7 @@
 
 class W_BaseSetObject(W_Root):
     typedef = None
+    exact_class_applevel_name = 'set-or-frozenset'
 
     def __init__(self, space, w_iterable=None):
         """Initialize the set by taking ownership of 'setdata'."""
@@ -489,6 +490,12 @@
 
 
 class W_SetObject(W_BaseSetObject):
+
+    #overridden here so the error is reported correctly
+    def __init__(self, space, w_iterable=None):
+        """Initialize the set by taking ownership of 'setdata'."""
+        W_BaseSetObject.__init__(self, space, w_iterable)
+
     def _newobj(self, space, w_iterable):
         """Make a new set by taking ownership of 'w_iterable'."""
         return W_SetObject(space, w_iterable)
@@ -504,7 +511,7 @@
 
 Build an unordered collection.""",
     __new__ = gateway.interp2app(W_SetObject.descr_new),
-    __init__ = gateway.interp2app(W_BaseSetObject.descr_init),
+    __init__ = gateway.interp2app(W_SetObject.descr_init),
     __repr__ = gateway.interp2app(W_BaseSetObject.descr_repr),
     __hash__ = None,
 
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -1050,3 +1050,10 @@
         id_a = id(a)
         assert a is not str(a, 'latin1')
         assert id_a != id_b
+
+    def test_error_message_wrong_self(self):
+        e = raises(TypeError, bytes.upper, 42)
+        assert "bytes" in str(e.value)
+        if hasattr(bytes.upper, 'im_func'):
+            e = raises(TypeError, bytes.upper.im_func, 42)
+            assert "'bytes'" in str(e.value)
diff --git a/pypy/objspace/std/test/test_intobject.py b/pypy/objspace/std/test/test_intobject.py
--- a/pypy/objspace/std/test/test_intobject.py
+++ b/pypy/objspace/std/test/test_intobject.py
@@ -800,6 +800,15 @@
             assert 567 .__round__(None) == 567    # fails on CPython
 
 
+    def test_error_message_wrong_self(self):
+        unboundmeth = int.__str__
+        e = raises(TypeError, unboundmeth, "!")
+        assert "int" in str(e.value)
+        if hasattr(unboundmeth, 'im_func'):
+            e = raises(TypeError, unboundmeth.im_func, "!")
+            assert "'int'" in str(e.value)
+
+
 class AppTestIntShortcut(AppTestInt):
     spaceconfig = {"objspace.std.intshortcut": True}
 
diff --git a/pypy/objspace/std/test/test_longobject.py b/pypy/objspace/std/test/test_longobject.py
--- a/pypy/objspace/std/test/test_longobject.py
+++ b/pypy/objspace/std/test/test_longobject.py
@@ -511,3 +511,11 @@
         assert a is not b
         b -= 1
         assert a is b
+
+    def test_error_message_wrong_self(self):
+        unboundmeth = long.__str__
+        e = raises(TypeError, unboundmeth, 42)
+        assert "int" in str(e.value)
+        if hasattr(unboundmeth, 'im_func'):
+            e = raises(TypeError, unboundmeth.im_func, 42)
+            assert "'int'" in str(e.value)
diff --git a/pypy/objspace/std/test/test_setobject.py b/pypy/objspace/std/test/test_setobject.py
--- a/pypy/objspace/std/test/test_setobject.py
+++ b/pypy/objspace/std/test/test_setobject.py
@@ -1118,3 +1118,15 @@
     def test_unicode_bug_in_listview_utf8(self):
         l1 = set(u'\u1234\u2345')
         assert l1 == set([u'\u1234', '\u2345'])
+
+    def test_frozenset_init_does_nothing(self):
+        f = frozenset([1, 2, 3])
+        f.__init__(4, 5, 6)
+        assert f == frozenset([1, 2, 3])
+
+    def test_error_message_wrong_self(self):
+        e = raises(TypeError, frozenset.copy, 42)
+        assert "frozenset" in str(e.value)
+        if hasattr(frozenset.copy, 'im_func'):
+            e = raises(TypeError, frozenset.copy.im_func, 42)
+            assert "'set-or-frozenset'" in str(e.value)
diff --git a/pypy/objspace/std/test/test_tupleobject.py b/pypy/objspace/std/test/test_tupleobject.py
--- a/pypy/objspace/std/test/test_tupleobject.py
+++ b/pypy/objspace/std/test/test_tupleobject.py
@@ -438,3 +438,12 @@
         assert (() != object()) is True
         assert ((1,) != object()) is True
         assert ((1, 2) != object()) is True
+
+
+    def test_error_message_wrong_self(self):
+        unboundmeth = tuple.__hash__
+        e = raises(TypeError, unboundmeth, 42)
+        assert "tuple" in str(e.value)
+        if hasattr(unboundmeth, 'im_func'):
+            e = raises(TypeError, unboundmeth.im_func, 42)
+            assert "'tuple'" in str(e.value)
\ No newline at end of file

From pypy.commits at gmail.com  Wed Oct 16 02:06:42 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 15 Oct 2019 23:06:42 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: remove python2 test
Message-ID: <5da6b372.1c69fb81.dbc1.4675@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r97785:58de798e3809
Date: 2019-10-16 09:06 +0300
http://bitbucket.org/pypy/pypy/changeset/58de798e3809/

Log:	remove python2 test

diff --git a/pypy/objspace/std/test/test_longobject.py b/pypy/objspace/std/test/test_longobject.py
--- a/pypy/objspace/std/test/test_longobject.py
+++ b/pypy/objspace/std/test/test_longobject.py
@@ -512,10 +512,3 @@
         b -= 1
         assert a is b
 
-    def test_error_message_wrong_self(self):
-        unboundmeth = long.__str__
-        e = raises(TypeError, unboundmeth, 42)
-        assert "int" in str(e.value)
-        if hasattr(unboundmeth, 'im_func'):
-            e = raises(TypeError, unboundmeth.im_func, 42)
-            assert "'int'" in str(e.value)

From pypy.commits at gmail.com  Wed Oct 16 08:04:55 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 16 Oct 2019 05:04:55 -0700 (PDT)
Subject: [pypy-commit] pypy default: small improvement: if the map can't be
 promoted (eg when using setattr), check
Message-ID: <5da70767.1c69fb81.84108.1965@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r97786:292b84d69855
Date: 2019-10-16 14:03 +0200
http://bitbucket.org/pypy/pypy/changeset/292b84d69855/

Log:	small improvement: if the map can't be promoted (eg when using
	setattr), check first whether the attribute goes into the first five
	slots, or into the extension array

diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -692,10 +692,12 @@
             return unerase_item(erased)
 
         def _mapdict_write_storage(self, storageindex, value):
-            for i in rangenmin1:
-                if storageindex == i:
-                    setattr(self, "_value%s" % i, value)
-                    return
+            assert storageindex >= 0
+            if storageindex < nmin1:
+                for i in rangenmin1:
+                    if storageindex == i:
+                        setattr(self, "_value%s" % i, value)
+                        return
             if self._has_storage_list():
                 self._mapdict_get_storage_list()[storageindex - nmin1] = value
                 return

From pypy.commits at gmail.com  Wed Oct 16 08:04:57 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 16 Oct 2019 05:04:57 -0700 (PDT)
Subject: [pypy-commit] pypy default: add two get_printable_locations
Message-ID: <5da70769.1c69fb81.c60fe.af50@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r97787:6050f7ce8f45
Date: 2019-10-16 14:03 +0200
http://bitbucket.org/pypy/pypy/changeset/6050f7ce8f45/

Log:	add two get_printable_locations

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -22,9 +22,13 @@
 
 __all__ = ['ObjSpace', 'OperationError', 'W_Root']
 
+def get_printable_location(tp):
+    return "unpackiterable: %s" % (tp, )
+
 unpackiterable_driver = jit.JitDriver(name='unpackiterable',
                                       greens=['tp'],
-                                      reds=['items', 'w_iterator'])
+                                      reds=['items', 'w_iterator'],
+                                      get_printable_location=get_printable_location)
 
 
 class W_Root(object):
diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -1664,9 +1664,13 @@
     w_set.sstorage = w_set.strategy.get_storage_from_list(iterable_w)
 
 
+def get_printable_location(tp, strategy):
+    return "create_set: %s %s" % (tp, strategy)
+
 create_set_driver = jit.JitDriver(name='create_set',
                                   greens=['tp', 'strategy'],
-                                  reds='auto')
+                                  reds='auto',
+                                  get_printable_location=get_printable_location)
 
 def _create_from_iterable(space, w_set, w_iterable):
     w_set.strategy = strategy = space.fromcache(EmptySetStrategy)

From pypy.commits at gmail.com  Wed Oct 16 08:45:07 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 16 Oct 2019 05:45:07 -0700 (PDT)
Subject: [pypy-commit] pypy record-known-result: these four operations are
 removed by the branch
Message-ID: <5da710d3.1c69fb81.3ff54.c5df@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: record-known-result
Changeset: r97788:4315f8c9ff0f
Date: 2019-10-16 14:17 +0200
http://bitbucket.org/pypy/pypy/changeset/4315f8c9ff0f/

Log:	these four operations are removed by the branch

diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -256,13 +256,9 @@
             i82 = int_ge(0, i77)
             guard_false(i82, descr=...)
             i85 = call_i(ConstClass(next_codepoint_pos_dont_look_inside), p78, 0, descr=...)
-            i86 = int_gt(i85, i79)
-            guard_false(i86, descr=...)
             i88 = int_ge(1, i77)
             guard_false(i88, descr=...)
             i90 = call_i(ConstClass(next_codepoint_pos_dont_look_inside), p78, i85, descr=...)
-            i91 = int_gt(i90, i79)
-            guard_false(i91, descr=...)
             i92 = int_sub(i90, i85)
             i94 = int_add(-1, i77)
             i96 = call_i(ConstClass(prev_codepoint_pos_dont_look_inside), p78, i79, descr=...)

From pypy.commits at gmail.com  Wed Oct 16 08:45:10 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 16 Oct 2019 05:45:10 -0700 (PDT)
Subject: [pypy-commit] pypy record-known-result: merge default
Message-ID: <5da710d6.1c69fb81.9326f.bc97@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: record-known-result
Changeset: r97789:f1e552055e69
Date: 2019-10-16 14:17 +0200
http://bitbucket.org/pypy/pypy/changeset/f1e552055e69/

Log:	merge default

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -50,4 +50,10 @@
 de061d87e39c7df4e436974096d7982c676a859d release-pypy3.6-v7.1.0
 784b254d669919c872a505b807db8462b6140973 release-pypy3.6-v7.1.1
 8cdda8b8cdb8ff29d9e620cccd6c5edd2f2a23ec release-pypy2.7-v7.1.1
-
+85dae4fd5c234b482feff834c73e089872194541 release-pypy2.7-v7.2.0rc0
+7ffb92269488f37c707ce66076f50ffd8613f8e2 release-pypy3.6-v7.2.0rc0
+4d6761df14ffd6f38450f183ac1fad32c946c21b release-pypy3.6-v7.2.0rc1
+5da45ced70e515f94686be0df47c59abd1348ebc release-pypy3.6-v7.2.0rc2
+4a68d8d3d2fc1faec2e83bcb4d28559099092574 release-pypy2.7-v7.2.0rc2
+4a68d8d3d2fc1faec2e83bcb4d28559099092574 release-pypy2.7-v7.2.0
+5da45ced70e515f94686be0df47c59abd1348ebc release-pypy3.6-v7.2.0
diff --git a/lib-python/2.7/test/test_dictviews.py b/lib-python/2.7/test/test_dictviews.py
--- a/lib-python/2.7/test/test_dictviews.py
+++ b/lib-python/2.7/test/test_dictviews.py
@@ -182,7 +182,7 @@
 
     def test_deeply_nested_repr(self):
         d = {}
-        for i in range(sys.getrecursionlimit() + 100):
+        for i in range(sys.getrecursionlimit() + 200):
             d = {42: d.viewvalues()}
         self.assertRaises(RuntimeError, repr, d)
 
diff --git a/pypy/doc/commandline_ref.rst b/pypy/doc/commandline_ref.rst
--- a/pypy/doc/commandline_ref.rst
+++ b/pypy/doc/commandline_ref.rst
@@ -9,3 +9,4 @@
 
    man/pypy.1.rst
    man/pypy3.1.rst
+   jit_help.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -42,7 +42,8 @@
 -------------------------------
 
 .. toctree::
-   whatsnew-pypy3-head.rst
+    whatsnew-pypy3-head.rst
+    whatsnew-pypy3-7.1.0.rst
 
 CPython 3.5 compatible versions
 -------------------------------
@@ -50,6 +51,8 @@
 .. toctree::
 
    whatsnew-pypy3-7.0.0.rst
+   whatsnew-pypy3-6.0.0.rst
+   whatsnew-pypy3-5.10.0.rst
    whatsnew-pypy3-5.9.0.rst
    whatsnew-pypy3-5.8.0.rst
    whatsnew-pypy3-5.7.0.rst
@@ -62,10 +65,4 @@
    whatsnew-pypy3-5.5.0.rst
    whatsnew-pypy3-5.1.1-alpha1.rst
 
-CPython 3.2 compatible versions
--------------------------------
 
-.. toctree::
-
-   whatsnew-pypy3-2.4.0.rst
-   whatsnew-pypy3-2.3.1.rst
diff --git a/pypy/doc/jit_help.rst b/pypy/doc/jit_help.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/jit_help.rst
@@ -0,0 +1,78 @@
+========
+JIT help
+========
+
+.. note this is from ``pypy --jit help``
+
+Advanced JIT options
+====================
+
+``<pypy> --jit`` [*options*] where *options* is a comma-separated list of
+``OPTION=VALUE``:
+
+ decay=N
+    amount to regularly decay counters by (0=none, 1000=max) (default 40)
+
+ disable_unrolling=N
+    after how many operations we should not unroll (default 200)
+
+ enable_opts=N
+    INTERNAL USE ONLY (MAY NOT WORK OR LEAD TO CRASHES): optimizations to
+    enable, or all =
+    intbounds:rewrite:virtualize:string:pure:earlyforce:heap:unroll (default
+    all)
+
+ function_threshold=N
+    number of times a function must run for it to become traced from start
+    (default 1619)
+
+ inlining=N
+    inline python functions or not (1/0) (default 1)
+
+ loop_longevity=N
+    a parameter controlling how long loops will be kept before being freed,
+    an estimate (default 1000)
+
+ max_retrace_guards=N
+    number of extra guards a retrace can cause (default 15)
+
+ max_unroll_loops=N
+    number of extra unrollings a loop can cause (default 0)
+
+ max_unroll_recursion=N
+    how many levels deep to unroll a recursive function (default 7)
+
+ retrace_limit=N
+    how many times we can try retracing before giving up (default 0)
+
+ threshold=N
+    number of times a loop has to run for it to become hot (default 1039)
+
+ trace_eagerness=N
+    number of times a guard has to fail before we start compiling a bridge
+    (default 200)
+
+ trace_limit=N
+    number of recorded operations before we abort tracing with ABORT_TOO_LONG
+    (default 6000)
+
+ vec=N
+    turn on the vectorization optimization (vecopt). Supports x86 (SSE 4.1),
+    powerpc (SVX), s390x SIMD (default 0)
+
+ vec_all=N
+    try to vectorize trace loops that occur outside of the numpypy library
+    (default 0)
+
+ vec_cost=N
+    threshold for which traces to bail. Unpacking increases the counter,
+    vector operation decrease the cost (default 0)
+
+ off
+    turn off the JIT
+ help
+    print this page
+
+The :ref:`pypyjit<jit-hooks>` module can be used to control the JIT from inside
+pypy
+
diff --git a/pypy/doc/release-v7.2.0.rst b/pypy/doc/release-v7.2.0.rst
--- a/pypy/doc/release-v7.2.0.rst
+++ b/pypy/doc/release-v7.2.0.rst
@@ -216,6 +216,7 @@
 * Add more constants to `sysconfig``. Set ``MACOSX_DEPLOYMENT_TARGET`` for
   darwin (`issue 2994`_)
 * fix ``CBuffer.buffer_attach``
+* Add ``_PyDict_GetItemWithError`` (``PyDict_GetItemWithError`` on Python3)
 
 Python 3.6 only
 ---------------
@@ -307,6 +308,7 @@
 .. _33786 : https://bugs.python.org/issue33786
 .. _32270 : https://bugs.python.org/issue32270
 .. _28691 : https://bugs.python.org/issue28691
+.. _33729 : https://bugs.python.org/issue33729
 
 .. _opencv2: https://github.com/skvark/opencv-python/
 .. _`issue 2617`: https://bitbucket.com/pypy/pypy/issues/2617
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -3,11 +3,9 @@
 ==========================
 
 .. this is a revision shortly after release-pypy-7.2.0
-.. startrev: 78cd4acbcbec 
+.. startrev: a511d86377d6 
 
+.. branch: fix-descrmismatch-crash
 
-.. branch: json-decoder-maps
+Fix segfault when calling descr-methods with no arguments
 
-Much faster and more memory-efficient JSON decoding. The resulting
-dictionaries that come out of the JSON decoder have faster lookups too.
-
diff --git a/pypy/doc/whatsnew-pypy2-5.7.0.rst b/pypy/doc/whatsnew-pypy2-5.7.0.rst
--- a/pypy/doc/whatsnew-pypy2-5.7.0.rst
+++ b/pypy/doc/whatsnew-pypy2-5.7.0.rst
@@ -14,6 +14,7 @@
 changed in exactly the same way because of this fix.
 
 
+
 .. branch: rpython-error-to-systemerror
 
 Any uncaught RPython exception (from a PyPy bug) is turned into an
diff --git a/pypy/doc/whatsnew-pypy2-7.2.0.rst b/pypy/doc/whatsnew-pypy2-7.2.0.rst
--- a/pypy/doc/whatsnew-pypy2-7.2.0.rst
+++ b/pypy/doc/whatsnew-pypy2-7.2.0.rst
@@ -74,3 +74,9 @@
 .. branch: openssl-for-macos
 
 Update _ssl on macos to statically link to openssl-1.1.1c
+
+.. branch: json-decoder-maps
+
+Much faster and more memory-efficient JSON decoding. The resulting
+dictionaries that come out of the JSON decoder have faster lookups too.
+
diff --git a/pypy/doc/whatsnew-pypy3-2.3.1.rst b/pypy/doc/whatsnew-pypy3-2.3.1.rst
deleted file mode 100644
--- a/pypy/doc/whatsnew-pypy3-2.3.1.rst
+++ /dev/null
@@ -1,6 +0,0 @@
-=========================
-What's new in PyPy3 2.3.1
-=========================
-
-.. this is a revision shortly after pypy3-release-2.3.x
-.. startrev: 0137d8e6657d
diff --git a/pypy/doc/whatsnew-pypy3-2.4.0.rst b/pypy/doc/whatsnew-pypy3-2.4.0.rst
deleted file mode 100644
--- a/pypy/doc/whatsnew-pypy3-2.4.0.rst
+++ /dev/null
@@ -1,6 +0,0 @@
-=========================
-What's new in PyPy3 2.4.0
-=========================
-
-.. this is a revision shortly after pypy3-release-2.4.x
-.. startrev: 12b940544622
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -22,9 +22,13 @@
 
 __all__ = ['ObjSpace', 'OperationError', 'W_Root']
 
+def get_printable_location(tp):
+    return "unpackiterable: %s" % (tp, )
+
 unpackiterable_driver = jit.JitDriver(name='unpackiterable',
                                       greens=['tp'],
-                                      reds=['items', 'w_iterator'])
+                                      reds=['items', 'w_iterator'],
+                                      get_printable_location=get_printable_location)
 
 
 class W_Root(object):
@@ -419,6 +423,8 @@
 
 @specialize.memo()
 def wrappable_class_name(Class):
+    if 'exact_class_applevel_name' in Class.__dict__:
+        return Class.exact_class_applevel_name
     try:
         return Class.typedef.name
     except AttributeError:
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -709,6 +709,7 @@
                     self.func__args__ = func
                 elif unwrap_spec == [self_type, ObjSpace, Arguments]:
                     self.__class__ = BuiltinCodePassThroughArguments1
+                    self.descr_reqcls = self_type
                     miniglobals = {'func': func, 'self_type': self_type}
                     d = {}
                     source = """if 1:
@@ -754,10 +755,7 @@
         except DescrMismatch:
             if w_obj is not None:
                 args = args.prepend(w_obj)
-            return scope_w[0].descr_call_mismatch(space,
-                                                  self.descrmismatch_op,
-                                                  self.descr_reqcls,
-                                                  args)
+            return self._type_unwrap_mismatch(space, args)
         except Exception as e:
             self.handle_exception(space, e)
             w_result = None
@@ -765,6 +763,15 @@
             w_result = space.w_None
         return w_result
 
+    def _type_unwrap_mismatch(self, space, args):
+        w_obj = args.firstarg()
+        if w_obj is None:
+            raise oefmt(space.w_SystemError, "unexpected DescrMismatch error")
+        return w_obj.descr_call_mismatch(space,
+                                         self.descrmismatch_op,
+                                         self.descr_reqcls,
+                                         args)
+
     def handle_exception(self, space, e):
         try:
             if not we_are_translated():
@@ -787,10 +794,7 @@
         try:
             w_result = self.func__args__(space, args)
         except DescrMismatch:
-            return args.firstarg().descr_call_mismatch(space,
-                                                  self.descrmismatch_op,
-                                                  self.descr_reqcls,
-                                                  args)
+            return self._type_unwrap_mismatch(space, args)
         except Exception as e:
             self.handle_exception(space, e)
             w_result = None
@@ -808,10 +812,7 @@
         try:
             w_result = self.func__args__(space, w_obj, args)
         except DescrMismatch:
-            return args.firstarg().descr_call_mismatch(space,
-                                                  self.descrmismatch_op,
-                                                  self.descr_reqcls,
-                                                  args.prepend(w_obj))
+            return self._type_unwrap_mismatch(space, args.prepend(w_obj))
         except Exception as e:
             self.handle_exception(space, e)
             w_result = None
@@ -851,9 +852,7 @@
         try:
             w_result = self.fastfunc_1(space, w1)
         except DescrMismatch:
-            return w1.descr_call_mismatch(space,
-                                          self.descrmismatch_op,
-                                          self.descr_reqcls,
+            return self._type_unwrap_mismatch(space,
                                           Arguments(space, [w1]))
         except Exception as e:
             self.handle_exception(space, e)
@@ -877,9 +876,7 @@
         try:
             w_result = self.fastfunc_2(space, w1, w2)
         except DescrMismatch:
-            return w1.descr_call_mismatch(space,
-                                          self.descrmismatch_op,
-                                          self.descr_reqcls,
+            return self._type_unwrap_mismatch(space,
                                           Arguments(space, [w1, w2]))
         except Exception as e:
             self.handle_exception(space, e)
@@ -904,9 +901,7 @@
         try:
             w_result = self.fastfunc_3(space, w1, w2, w3)
         except DescrMismatch:
-            return w1.descr_call_mismatch(space,
-                                          self.descrmismatch_op,
-                                          self.descr_reqcls,
+            return self._type_unwrap_mismatch(space,
                                           Arguments(space, [w1, w2, w3]))
         except Exception as e:
             self.handle_exception(space, e)
@@ -932,9 +927,7 @@
         try:
             w_result = self.fastfunc_4(space, w1, w2, w3, w4)
         except DescrMismatch:
-            return w1.descr_call_mismatch(space,
-                                          self.descrmismatch_op,
-                                          self.descr_reqcls,
+            return self._type_unwrap_mismatch(space,
                                           Arguments(space,
                                                     [w1, w2, w3, w4]))
         except Exception as e:
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -966,6 +966,29 @@
         # white-box check for opt
         assert called[0] is args
 
+    def test_base_regular_descr_mismatch(self):
+        space = self.space
+
+        def f():
+            raise gateway.DescrMismatch
+
+        w_f = space.wrap(gateway.interp2app_temp(f,
+                         unwrap_spec=[]))
+        args = argument.Arguments(space, [])
+        space.raises_w(space.w_SystemError, space.call_args, w_f, args)
+
+    def test_pass_trough_arguments0_descr_mismatch(self):
+        space = self.space
+
+        def f(space, __args__):
+            raise gateway.DescrMismatch
+
+        w_f = space.wrap(gateway.interp2app_temp(f,
+                         unwrap_spec=[gateway.ObjSpace,
+                                      gateway.Arguments]))
+        args = argument.Arguments(space, [])
+        space.raises_w(space.w_SystemError, space.call_args, w_f, args)
+
 
 class AppTestKeywordsToBuiltinSanity(object):
     def test_type(self):
@@ -1005,3 +1028,26 @@
 
         d.update(**{clash: 33})
         dict.update(d, **{clash: 33})
+
+
+
+class AppTestFastPathCrash(object):
+    def setup_class(cls):
+        cls.w_runappdirect = cls.space.wrap(cls.runappdirect)
+
+    def test_fast_path_crash(self):
+        # issue bb-3091 crash in BuiltinCodePassThroughArguments0.funcrun
+        for obj in (dict, set):
+            with raises(TypeError) as excinfo:
+                if self.runappdirect:
+                    import platform
+                    if platform.python_implementation() == 'PyPy':
+                        msg_fmt = "%s instance as first argument (got %s"
+                    else:
+                        msg_fmt = "'%s' object but received a '%s'"
+                    obj.__init__(0)
+                else:
+                    msg_fmt = "'%s' object expected, got '%s'"
+                    obj.__init__.im_func(0)
+            msg = msg_fmt %(obj.__name__, 'int')
+            assert msg in str(excinfo.value)
diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -69,6 +69,9 @@
     # hit in the cache
     STRING_CACHE_USEFULNESS_FACTOR = 4
 
+    # don't make arbitrarily huge maps
+    MAX_MAP_SIZE = 100
+
 
     def __init__(self, space, s):
         self.space = space
@@ -368,7 +371,7 @@
                 return w_res
             elif ch == ',':
                 i = self.skip_whitespace(i)
-                if currmap.is_state_blocked():
+                if currmap.is_state_blocked() or nextindex > self.MAX_MAP_SIZE:
                     self.scratch.append(values_w)  # can reuse next time
                     dict_w = self._switch_to_dict(currmap, values_w, nextindex)
                     return self.decode_object_dict(i, start, dict_w)
diff --git a/pypy/module/_pypyjson/simd.py b/pypy/module/_pypyjson/simd.py
--- a/pypy/module/_pypyjson/simd.py
+++ b/pypy/module/_pypyjson/simd.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib import objectmodel, unroll
 from rpython.rlib.rarithmetic import r_uint, intmask, LONG_BIT
-from rpython.jit.backend.detect_cpu import autodetect
+from rpython.jit.backend.detect_cpu import autodetect, ProcessorAutodetectError
 
 # accelerators for string operations using simd on regular word sizes (*not*
 # SSE instructions). this style is sometimes called SWAR (SIMD Within A
@@ -15,8 +15,11 @@
     WORD_SIZE = 8
     EVERY_BYTE_ONE = 0x0101010101010101
     EVERY_BYTE_HIGHEST_BIT = 0x8080808080808080
-    if autodetect() == "x86-64":
-        USE_SIMD = True
+    try:
+        if autodetect() == "x86-64":
+            USE_SIMD = True
+    except ProcessorAutodetectError:
+        pass
 else:
     WORD_SIZE = 4
     EVERY_BYTE_ONE = 0x01010101
diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py
--- a/pypy/module/_pypyjson/test/test__pypyjson.py
+++ b/pypy/module/_pypyjson/test/test__pypyjson.py
@@ -469,6 +469,14 @@
         res = _pypyjson.loads(json)
         assert res == [{u'a': 1}, {u'a': 2}]
 
+    def test_huge_map(self):
+        import _pypyjson
+        import __pypy__
+        s = '{' + ",".join('"%s": %s' % (i, i) for i in range(200)) + '}'
+        res = _pypyjson.loads(s)
+        assert len(res) == 200
+        assert __pypy__.strategy(res) == "UnicodeDictStrategy"
+
     def test_tab_in_string_should_fail(self):
         import _pypyjson
         # http://json.org/JSON_checker/test/fail25.json
diff --git a/pypy/module/_weakref/interp__weakref.py b/pypy/module/_weakref/interp__weakref.py
--- a/pypy/module/_weakref/interp__weakref.py
+++ b/pypy/module/_weakref/interp__weakref.py
@@ -156,6 +156,8 @@
 
 
 class W_WeakrefBase(W_Root):
+    exact_class_applevel_name = 'weakref-or-proxy'
+
     def __init__(self, space, w_obj, w_callable):
         assert w_callable is not space.w_None    # should be really None
         self.space = space
diff --git a/pypy/module/_weakref/test/test_weakref.py b/pypy/module/_weakref/test/test_weakref.py
--- a/pypy/module/_weakref/test/test_weakref.py
+++ b/pypy/module/_weakref/test/test_weakref.py
@@ -543,3 +543,12 @@
         p1[42] = p2
         assert a1.setkey == 42
         assert a1.setvalue is p2
+
+    def test_error_message_wrong_self(self):
+        import _weakref
+        unboundmeth = _weakref.ref.__repr__
+        e = raises(TypeError, unboundmeth, 42)
+        assert "weakref" in str(e.value)
+        if hasattr(unboundmeth, 'im_func'):
+            e = raises(TypeError, unboundmeth.im_func, 42)
+            assert "'weakref-or-proxy'" in str(e.value)
diff --git a/pypy/module/cStringIO/interp_stringio.py b/pypy/module/cStringIO/interp_stringio.py
--- a/pypy/module/cStringIO/interp_stringio.py
+++ b/pypy/module/cStringIO/interp_stringio.py
@@ -7,6 +7,8 @@
 
 
 class W_InputOutputType(W_Root):
+    exact_class_applevel_name = "StringI-or-StringO"
+
     softspace = 0    # part of the file object API
 
     def descr___iter__(self):
diff --git a/pypy/module/cStringIO/test/test_interp_stringio.py b/pypy/module/cStringIO/test/test_interp_stringio.py
--- a/pypy/module/cStringIO/test/test_interp_stringio.py
+++ b/pypy/module/cStringIO/test/test_interp_stringio.py
@@ -204,3 +204,12 @@
         import cStringIO
         assert type(cStringIO.StringIO()) is cStringIO.OutputType
         assert type(cStringIO.StringIO('')) is cStringIO.InputType
+
+    def test_error_message_wrong_self(self):
+        import cStringIO
+        unboundmeth = cStringIO.InputType.close
+        e = raises(TypeError, unboundmeth, 42)
+        assert "StringI" in str(e.value)
+        if hasattr(unboundmeth, 'im_func'):
+            e = raises(TypeError, unboundmeth.im_func, 42)
+            assert "'StringI-or-StringO'" in str(e.value)
diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -80,6 +80,13 @@
     # XXX this is wrong with IntMutableCell.  Hope it works...
     return w_dict.getitem(w_key)
 
+ at cpython_api([PyObject, PyObject], PyObject, result_borrowed=True)
+def _PyDict_GetItemWithError(space, w_dict, w_key):
+    # Like PyDict_GetItem(), but doesn't swallow the error
+    if not isinstance(w_dict, W_DictMultiObject):
+        PyErr_BadInternalCall(space)
+    return w_dict.getitem(w_key)
+
 @cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)
 def PyDict_SetItem(space, w_dict, w_key, w_obj):
     if not isinstance(w_dict, W_DictMultiObject):
diff --git a/pypy/module/cpyext/test/test_dictobject.py b/pypy/module/cpyext/test/test_dictobject.py
--- a/pypy/module/cpyext/test/test_dictobject.py
+++ b/pypy/module/cpyext/test/test_dictobject.py
@@ -173,6 +173,26 @@
             ])
         assert module.dict_proxy({'a': 1, 'b': 2}) == 2
 
+    def test_getitemwitherror(self):
+        module = self.import_extension('foo', [
+            ("dict_getitem", "METH_VARARGS",
+             """
+             PyObject *d, *key, *result;
+             if (!PyArg_ParseTuple(args, "OO", &d, &key)) {
+                return NULL;
+             }
+             result = _PyDict_GetItemWithError(d, key);
+             if (result == NULL && !PyErr_Occurred())
+                Py_RETURN_NONE;
+             Py_XINCREF(result);
+             return result;
+             """)])
+        d = {'foo': 'bar'}
+        assert module.dict_getitem(d, 'foo') == 'bar'
+        assert module.dict_getitem(d, 'missing') is None
+        with raises(TypeError):
+            module.dict_getitem(d, [])
+
     def test_update(self):
         module = self.import_extension('foo', [
             ("update", "METH_VARARGS",
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -149,6 +149,9 @@
         is_root(w_obj)
         return NonConstant(False)
 
+    def utf8_len_w(self, space):
+        return NonConstant((NonConstant("utf8len_foobar"), NonConstant(14)))
+
     @not_rpython
     def unwrap(self, w_obj):
         raise NotImplementedError
diff --git a/pypy/objspace/fake/test/test_objspace.py b/pypy/objspace/fake/test/test_objspace.py
--- a/pypy/objspace/fake/test/test_objspace.py
+++ b/pypy/objspace/fake/test/test_objspace.py
@@ -1,4 +1,5 @@
-import py
+import pytest
+from rpython.rlib.nonconst import NonConstant
 from pypy.objspace.fake.objspace import FakeObjSpace, W_Root
 from pypy.interpreter.argument import Arguments
 from pypy.interpreter.typedef import TypeDef
@@ -63,8 +64,8 @@
     def test_is_true(self):
         space = self.space
         space.translates(lambda: space.is_true(W_Root()))
-        py.test.raises(AssertionError,
-                       space.translates, lambda: space.is_true(42))
+        with pytest.raises(AssertionError):
+            space.translates(lambda: space.is_true(42))
 
     def test_unpackiterable(self):
         space = self.space
@@ -79,3 +80,23 @@
         space = self.space
         space.translates(lambda: (space.get(W_Root(), W_Root()),
                                   space.get(W_Root(), W_Root(), W_Root())))
+
+    def test_bug_utf8_len_w(self):
+        space = self.space
+
+        class A(object):
+            pass
+
+        def f():
+            s = NonConstant('a')
+            w_s = space.newutf8(s, 1)
+            t, l = space.utf8_len_w(w_s)
+            a = A()
+            if l == 1:
+                a.x = 1
+            else:
+                raise Exception
+            return a.x
+        space.translates(f)
+
+
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -24,6 +24,7 @@
 
 class W_AbstractBytesObject(W_Root):
     __slots__ = ()
+    exact_class_applevel_name = 'str'
 
     def is_w(self, space, w_other):
         if not isinstance(w_other, W_AbstractBytesObject):
diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -692,10 +692,12 @@
             return unerase_item(erased)
 
         def _mapdict_write_storage(self, storageindex, value):
-            for i in rangenmin1:
-                if storageindex == i:
-                    setattr(self, "_value%s" % i, value)
-                    return
+            assert storageindex >= 0
+            if storageindex < nmin1:
+                for i in rangenmin1:
+                    if storageindex == i:
+                        setattr(self, "_value%s" % i, value)
+                        return
             if self._has_storage_list():
                 self._mapdict_get_storage_list()[storageindex - nmin1] = value
                 return
diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -20,6 +20,7 @@
 
 class W_BaseSetObject(W_Root):
     typedef = None
+    exact_class_applevel_name = 'set-or-frozenset'
 
     def __init__(self, space, w_iterable=None):
         """Initialize the set by taking ownership of 'setdata'."""
@@ -496,6 +497,12 @@
 
 
 class W_SetObject(W_BaseSetObject):
+
+    #overridden here so the error is reported correctly
+    def __init__(self, space, w_iterable=None):
+        """Initialize the set by taking ownership of 'setdata'."""
+        W_BaseSetObject.__init__(self, space, w_iterable)
+
     def _newobj(self, space, w_iterable):
         """Make a new set by taking ownership of 'w_iterable'."""
         if type(self) is W_SetObject:
@@ -516,7 +523,7 @@
 
 Build an unordered collection.""",
     __new__ = gateway.interp2app(W_SetObject.descr_new),
-    __init__ = gateway.interp2app(W_BaseSetObject.descr_init),
+    __init__ = gateway.interp2app(W_SetObject.descr_init),
     __repr__ = gateway.interp2app(W_BaseSetObject.descr_repr),
     __hash__ = None,
     __cmp__ = gateway.interp2app(W_BaseSetObject.descr_cmp),
@@ -1657,9 +1664,13 @@
     w_set.sstorage = w_set.strategy.get_storage_from_list(iterable_w)
 
 
+def get_printable_location(tp, strategy):
+    return "create_set: %s %s" % (tp, strategy)
+
 create_set_driver = jit.JitDriver(name='create_set',
                                   greens=['tp', 'strategy'],
-                                  reds='auto')
+                                  reds='auto',
+                                  get_printable_location=get_printable_location)
 
 def _create_from_iterable(space, w_set, w_iterable):
     w_set.strategy = strategy = space.fromcache(EmptySetStrategy)
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -933,3 +933,10 @@
     def test_add(self):
         assert 'abc' + 'abc' == 'abcabc'
         assert isinstance('abc' + u'\u03a3', unicode)
+
+    def test_error_message_wrong_self(self):
+        e = raises(TypeError, bytes.upper, 42)
+        assert "str" in str(e.value)
+        if hasattr(bytes.upper, 'im_func'):
+            e = raises(TypeError, bytes.upper.im_func, 42)
+            assert "'str'" in str(e.value)
diff --git a/pypy/objspace/std/test/test_intobject.py b/pypy/objspace/std/test/test_intobject.py
--- a/pypy/objspace/std/test/test_intobject.py
+++ b/pypy/objspace/std/test/test_intobject.py
@@ -684,6 +684,15 @@
         x = int(-sys.maxint)
         assert x.__rsub__(2) == (2 + sys.maxint)
 
+    def test_error_message_wrong_self(self):
+        unboundmeth = int.__str__
+        e = raises(TypeError, unboundmeth, "!")
+        assert "int" in str(e.value)
+        if hasattr(unboundmeth, 'im_func'):
+            e = raises(TypeError, unboundmeth.im_func, "!")
+            assert "'int'" in str(e.value)
+
+
 class AppTestIntShortcut(AppTestInt):
     spaceconfig = {"objspace.std.intshortcut": True}
 
diff --git a/pypy/objspace/std/test/test_longobject.py b/pypy/objspace/std/test/test_longobject.py
--- a/pypy/objspace/std/test/test_longobject.py
+++ b/pypy/objspace/std/test/test_longobject.py
@@ -450,3 +450,10 @@
         expected = (2 << (size * 4)) // 3
         assert long(n, 16) == expected
 
+    def test_error_message_wrong_self(self):
+        unboundmeth = long.__str__
+        e = raises(TypeError, unboundmeth, 42)
+        assert "long" in str(e.value)
+        if hasattr(unboundmeth, 'im_func'):
+            e = raises(TypeError, unboundmeth.im_func, 42)
+            assert "'long'" in str(e.value)
diff --git a/pypy/objspace/std/test/test_setobject.py b/pypy/objspace/std/test/test_setobject.py
--- a/pypy/objspace/std/test/test_setobject.py
+++ b/pypy/objspace/std/test/test_setobject.py
@@ -1037,3 +1037,15 @@
            raise ValueError
            yield 1
         raises(ValueError, set, f())
+
+    def test_frozenset_init_does_nothing(self):
+        f = frozenset([1, 2, 3])
+        f.__init__(4, 5, 6)
+        assert f == frozenset([1, 2, 3])
+
+    def test_error_message_wrong_self(self):
+        e = raises(TypeError, frozenset.copy, 42)
+        assert "frozenset" in str(e.value)
+        if hasattr(frozenset.copy, 'im_func'):
+            e = raises(TypeError, frozenset.copy.im_func, 42)
+            assert "'set-or-frozenset'" in str(e.value)
diff --git a/pypy/objspace/std/test/test_tupleobject.py b/pypy/objspace/std/test/test_tupleobject.py
--- a/pypy/objspace/std/test/test_tupleobject.py
+++ b/pypy/objspace/std/test/test_tupleobject.py
@@ -454,3 +454,11 @@
             (4.1, 2.3), (3.6, 4.8)]
         assert specialized_zip_2_lists(["foo", "bar"], [6, 2]) == [
             ("foo", 6), ("bar", 2)]
+
+    def test_error_message_wrong_self(self):
+        unboundmeth = tuple.__hash__
+        e = raises(TypeError, unboundmeth, 42)
+        assert "tuple" in str(e.value)
+        if hasattr(unboundmeth, 'im_func'):
+            e = raises(TypeError, unboundmeth.im_func, 42)
+            assert "'tuple'" in str(e.value)
diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,15 +1,18 @@
 # Edit these appropriately before running this script
 pmaj=2  # python main version: 2 or 3
 pmin=7  # python minor version
-exe=pypy3 # pypy3 or pypy
 maj=7
 min=2
-rev=0
+rev=0rc2
 
+case $pmaj in
+    "2") exe=pypy;;
+    "3") exe=pypy3;;
+    *) echo invalid pmaj=$pmaj; exit 1;;
+esac
 
 branchname=release-pypy$pmaj.$pmin-v$maj.x # ==OR== release-v$maj.x  # ==OR== release-v$maj.$min.x
-tagname=release-candidate-pypy$pmaj.$pmin-v$maj.$min.$rev  # ==OR== release-$maj.$min
-# tagname=release-pypy$pmaj.$pmin-v$maj.$min.$rev  # ==OR== release-$maj.$min
+tagname=release-pypy$pmaj.$pmin-v$maj.$min.$rev  # ==OR== release-$maj.$min
 
 echo checking hg log -r $branchname
 hg log -r $branchname || exit 1
diff --git a/requirements.txt b/requirements.txt
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,5 +6,5 @@
 vmprof>=0.4.10; 'x86' in platform.machine #skip arm, s390x
 
 # hypothesis is used for test generation on untranslated tests
-hypothesis
+hypothesis<4.40
 enum34>=1.1.2
diff --git a/rpython/jit/metainterp/quasiimmut.py b/rpython/jit/metainterp/quasiimmut.py
--- a/rpython/jit/metainterp/quasiimmut.py
+++ b/rpython/jit/metainterp/quasiimmut.py
@@ -5,6 +5,7 @@
 from rpython.jit.metainterp.history import (
     AbstractDescr, ConstPtr, ConstInt, ConstFloat)
 from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop
 
 
 def get_mutate_field_name(fieldname):
@@ -26,12 +27,14 @@
     return qmut
 
 def make_invalidation_function(STRUCT, mutatefieldname):
-    #
+    # fake a repr
+    descr_repr = "FieldDescr(%s, '%s')" % (STRUCT.TO, mutatefieldname)
+
     def _invalidate_now(p):
         qmut_ptr = getattr(p, mutatefieldname)
         setattr(p, mutatefieldname, lltype.nullptr(rclass.OBJECT))
         qmut = cast_base_ptr_to_instance(QuasiImmut, qmut_ptr)
-        qmut.invalidate()
+        qmut.invalidate(descr_repr)
     _invalidate_now._dont_inline_ = True
     #
     def invalidation(p):
@@ -45,7 +48,7 @@
     if qmut_ref:
         cpu.bh_setfield_gc_r(p, ConstPtr.value, mutatefielddescr)
         qmut = cast_gcref_to_instance(QuasiImmut, qmut_ref)
-        qmut.invalidate()
+        qmut.invalidate(mutatefielddescr.repr_of_descr())
 
 
 class QuasiImmut(object):
@@ -78,7 +81,8 @@
         # already invalidated; see below
         self.compress_limit = (len(self.looptokens_wrefs) + 15) * 2
 
-    def invalidate(self):
+    def invalidate(self, descr_repr=None):
+        debug_start("jit-invalidate-quasi-immutable")
         # When this is called, all the loops that we record become
         # invalid: all GUARD_NOT_INVALIDATED in these loops (and
         # in attached bridges) must now fail.
@@ -87,9 +91,11 @@
             return
         wrefs = self.looptokens_wrefs
         self.looptokens_wrefs = []
+        invalidated = 0
         for wref in wrefs:
             looptoken = wref()
             if looptoken is not None:
+                invalidated += 1
                 looptoken.invalidated = True
                 self.cpu.invalidate_loop(looptoken)
                 # NB. we must call cpu.invalidate_loop() even if
@@ -100,6 +106,8 @@
                 if not we_are_translated():
                     self.cpu.stats.invalidated_token_numbers.add(
                         looptoken.number)
+        debug_print("fieldname", descr_repr or "<unknown>", "invalidated", invalidated)
+        debug_stop("jit-invalidate-quasi-immutable")
 
 
 class QuasiImmutDescr(AbstractDescr):
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -1903,7 +1903,9 @@
                 if MultiByteToWideChar(CP_ACP, flags,
                                        dataptr, size, buf.raw, usize) == 0:
                     _decode_mbcs_error(s, errorhandler)
-                return buf.str(usize), size
+                ret = buf.str(usize)
+                assert ret is not None
+                return ret, size
 
     def unicode_encode_mbcs(s, size, errors, errorhandler=None,
                             force_replace=True):
diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
--- a/rpython/translator/revdb/src-revdb/revdb_include.h
+++ b/rpython/translator/revdb/src-revdb/revdb_include.h
@@ -285,6 +285,8 @@
 #define OP_GC_RAWREFCOUNT_NEXT_DEAD(r)   \
     r = rpy_reverse_db_rawrefcount_next_dead()
 
+#define OP_GC_INCREASE_ROOT_STACK_DEPTH(depth, r)   /* nothing */
+
 
 RPY_EXTERN void rpy_reverse_db_flush(void);  /* must be called with the lock */
 RPY_EXTERN void rpy_reverse_db_fetch(const char *file, int line);

From pypy.commits at gmail.com  Wed Oct 16 08:45:12 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 16 Oct 2019 05:45:12 -0700 (PDT)
Subject: [pypy-commit] pypy record-known-result: re-decoding a utf-8 encoded
 byte string needs to do nothing
Message-ID: <5da710d8.1c69fb81.a4e43.edf7@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: record-known-result
Changeset: r97790:270b9c958d32
Date: 2019-10-16 14:44 +0200
http://bitbucket.org/pypy/pypy/changeset/270b9c958d32/

Log:	re-decoding a utf-8 encoded byte string needs to do nothing

diff --git a/dotviewer/graphclient.py b/dotviewer/graphclient.py
--- a/dotviewer/graphclient.py
+++ b/dotviewer/graphclient.py
@@ -95,8 +95,8 @@
     except EOFError:
         ioerror = ioerror or IOError("connection unexpectedly closed "
                                      "(graphserver crash?)")
-    #if ioerror is not None:
-    #    raise ioerror
+    if ioerror is not None:
+        raise ioerror
 
 def send_error(io, e):
     try:
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -289,3 +289,23 @@
             i57 = int_sub(i52, i56)
             i59 = int_sub(i38, 1)
         ''')
+
+    def test_decode_encode(self):
+        log = self.run("""
+        def main(n):
+            global s
+            u = b'ab\xc3\xa4\xf0\x9f\x91\xa9\xe2\x80\x8d\xf0\x9f\x91\xa9\xe2\x80\x8d\xf0\x9f\x91\xa7\xe2\x80\x8d\xf0\x9f\x91\xa6'.decode("utf-8")
+            count = 0
+            for i in range(n):
+                b = (u + unicode(i)).encode("utf-8")
+                u = b.decode("utf-8") # ID: decode
+                count += 1
+            return count
+        """, [10000])
+        loop, = log.loops_by_filename(self.filepath)
+        # No call to _check_utf8 is necessary, because the bytes come from
+        # W_UnicodeObject.utf8_w.
+        assert loop.match_by_id('decode', '''
+            i95 = int_ge(i86, 0)
+            guard_true(i95, descr=...)
+        ''')

From pypy.commits at gmail.com  Wed Oct 16 08:47:32 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 16 Oct 2019 05:47:32 -0700 (PDT)
Subject: [pypy-commit] pypy py3.7: Merged in
 Yannick_Jadoul/pypy/py3.7-pep553 (pull request #671)
Message-ID: <5da71164.1c69fb81.784e4.cf64@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.7
Changeset: r97793:690fff498e45
Date: 2019-10-16 12:45 +0000
http://bitbucket.org/pypy/pypy/changeset/690fff498e45/

Log:	Merged in Yannick_Jadoul/pypy/py3.7-pep553 (pull request #671)

	PEP 533 implementation

diff --git a/pypy/module/__builtin__/app_breakpoint.py b/pypy/module/__builtin__/app_breakpoint.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__builtin__/app_breakpoint.py
@@ -0,0 +1,16 @@
+# NOT_RPYTHON
+"""
+Plain Python definition of the builtin breakpoint function.
+"""
+
+import sys
+
+def breakpoint(*args, **kwargs):
+    """Call sys.breakpointhook(*args, **kws).  sys.breakpointhook() must accept
+whatever arguments are passed.
+
+By default, this drops you into the pdb debugger."""
+
+    if not hasattr(sys, 'breakpointhook'):
+        raise RuntimeError('lost sys.breakpointhook')
+    return sys.breakpointhook(*args, **kwargs)
diff --git a/pypy/module/__builtin__/moduledef.py b/pypy/module/__builtin__/moduledef.py
--- a/pypy/module/__builtin__/moduledef.py
+++ b/pypy/module/__builtin__/moduledef.py
@@ -23,6 +23,8 @@
         'bin'           : 'app_operation.bin',
         'oct'           : 'app_operation.oct',
         'hex'           : 'app_operation.hex',
+
+        'breakpoint'    : 'app_breakpoint.breakpoint',
     }
 
     interpleveldefs = {
diff --git a/pypy/module/__builtin__/test/test_breakpoint.py b/pypy/module/__builtin__/test/test_breakpoint.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__builtin__/test/test_breakpoint.py
@@ -0,0 +1,158 @@
+class AppTestBreakpoint:
+    def setup_class(cls):
+        cls.w_import_mock_pdb = cls.space.appexec([], """():
+            import sys
+            from contextlib import contextmanager
+            from types import ModuleType
+
+            @contextmanager
+            def w_import_mock_pdb():
+                try:
+                    mock_pdb = ModuleType('pdb')
+                    mock_pdb.set_trace = None
+                    sys.modules['pdb'] = mock_pdb
+                    yield mock_pdb
+                finally:
+                    del sys.modules['pdb']
+
+            return w_import_mock_pdb
+        """)
+
+        cls.w_mock_function = cls.space.appexec([], """():
+            from contextlib import contextmanager
+
+            class MockedCallable:
+                def __init__(self):
+                    self.called = 0
+
+                def __call__(self, *args, **kwargs):
+                    self.called += 1
+                    self.last_call = (args, kwargs)
+
+            @contextmanager
+            def mock_function(scope, attr, delete=False):
+                old = getattr(scope, attr)
+                try:
+                    if not delete:
+                        new = MockedCallable()
+                        setattr(scope, attr, new)
+                        yield new
+                    else:
+                        delattr(scope, attr)
+                        yield
+                finally:
+                    setattr(scope, attr, old)
+
+            return mock_function
+        """)
+
+
+    def test_default(self):
+        import sys
+        assert sys.breakpointhook is sys.__breakpointhook__
+
+        import os
+        assert 'PYTHONBREAKPOINT' not in os.environ
+
+        with self.import_mock_pdb() as pdb:
+            with self.mock_function(pdb, 'set_trace') as mocked:
+                breakpoint()
+                assert mocked.called == 1
+
+
+    def test_args_kwargs(self):
+        import sys
+        with self.import_mock_pdb() as pdb:
+            with self.mock_function(pdb, 'set_trace') as mocked:
+                breakpoint(1, 2, 3, x=4, y=5)
+                assert mocked.called == 1
+                assert mocked.last_call == ((1, 2, 3), {'x': 4, 'y': 5})
+
+
+    def test_breakpointhook(self):
+        import sys
+        with self.import_mock_pdb() as pdb:
+            with self.mock_function(sys, 'breakpointhook') as mocked, \
+                self.mock_function(pdb, 'set_trace') as mocked_pdb:
+                breakpoint()
+                assert mocked.called == 1
+                assert mocked_pdb.called == 0
+
+            with self.mock_function(pdb, 'set_trace') as mocked_pdb:
+                breakpoint()
+                assert mocked_pdb.called == 1
+
+
+    def test_breakpointhook_lost(self):
+        import sys
+        with self.import_mock_pdb() as pdb:
+            with self.mock_function(sys, 'breakpointhook', delete=True):
+                with raises(RuntimeError) as excinfo:
+                    breakpoint()
+                assert str(excinfo.value) == "lost sys.breakpointhook"
+
+
+    def test_env_default(self):
+        import os
+        try:
+            os.environ['PYTHONBREAKPOINT'] = ""
+
+            import sys
+            with self.import_mock_pdb() as pdb:
+                with self.mock_function(pdb, 'set_trace') as mocked:
+                    breakpoint()
+                    assert mocked.called == 1
+        finally:
+            del os.environ['PYTHONBREAKPOINT']
+
+
+    def test_env_disable(self):
+        import os
+        try:
+            os.environ['PYTHONBREAKPOINT'] = "0"
+
+            import sys
+            with self.import_mock_pdb() as pdb:
+                with self.mock_function(pdb, 'set_trace') as mocked:
+                    breakpoint()
+                    assert mocked.called == 0
+
+                with self.mock_function(sys, 'breakpointhook') as mocked:
+                    breakpoint()
+                    assert mocked.called == 1
+        finally:
+            del os.environ['PYTHONBREAKPOINT']
+
+
+    def test_env_other(self):
+        import os
+        try:
+            os.environ['PYTHONBREAKPOINT'] = 'sys.exit'
+
+            import sys
+            with self.import_mock_pdb() as pdb:
+                with self.mock_function(sys, 'exit') as mocked, \
+                     self.mock_function(pdb, 'set_trace') as mocked_pdb:
+                    breakpoint()
+                    assert mocked.called == 1
+                    assert mocked_pdb.called == 0
+        finally:
+            del os.environ['PYTHONBREAKPOINT']
+
+
+    def test_env_nonexistent(self):
+        import os
+        import warnings
+        try:
+            os.environ['PYTHONBREAKPOINT'] = 'blah.bleh'
+
+            import sys
+            with self.import_mock_pdb() as pdb:
+                with self.mock_function(pdb, 'set_trace') as mocked_pdb, \
+                     warnings.catch_warnings(record=True) as w:
+                    breakpoint()
+                    assert mocked_pdb.called == 0
+                    assert len(w) == 1
+                    assert str(w[-1].message) == 'Ignoring unimportable $PYTHONBREAKPOINT: "blah.bleh"'
+        finally:
+            del os.environ['PYTHONBREAKPOINT']
diff --git a/pypy/module/sys/app.py b/pypy/module/sys/app.py
--- a/pypy/module/sys/app.py
+++ b/pypy/module/sys/app.py
@@ -68,6 +68,31 @@
     except:
         return False    # got an exception again... ignore, report the original
 
+def breakpointhook(*args, **kwargs):
+    """This hook function is called by built-in breakpoint()."""
+
+    import importlib, os, warnings
+
+    hookname = os.getenv('PYTHONBREAKPOINT')
+    if hookname is None or len(hookname) == 0:
+        hookname = 'pdb.set_trace'
+    elif hookname == '0':
+        return None
+    modname, dot, funcname = hookname.rpartition('.')
+    if dot == '':
+        modname = 'builtins'
+
+    try:
+        module = importlib.import_module(modname)
+        hook = getattr(module, funcname)
+    except:
+        warnings.warn(
+            'Ignoring unimportable $PYTHONBREAKPOINT: "{}"'.format(hookname),
+            RuntimeWarning)
+        return None
+
+    return hook(*args, **kwargs)
+
 def exit(exitcode=None):
     """Exit the interpreter by raising SystemExit(exitcode).
 If the exitcode is omitted or None, it defaults to zero (i.e., success).
diff --git a/pypy/module/sys/moduledef.py b/pypy/module/sys/moduledef.py
--- a/pypy/module/sys/moduledef.py
+++ b/pypy/module/sys/moduledef.py
@@ -109,6 +109,8 @@
     appleveldefs = {
         'excepthook'            : 'app.excepthook',
         '__excepthook__'        : 'app.excepthook',
+        'breakpointhook'        : 'app.breakpointhook',
+        '__breakpointhook__'    : 'app.breakpointhook',
         'exit'                  : 'app.exit',
         'callstats'             : 'app.callstats',
         'copyright'             : 'app.copyright_str',

From pypy.commits at gmail.com  Wed Oct 16 08:48:17 2019
From: pypy.commits at gmail.com (Yannick_Jadoul)
Date: Wed, 16 Oct 2019 05:48:17 -0700 (PDT)
Subject: [pypy-commit] pypy py3.7-pep553: Implemented breakpoint and
 sys.breakpointhook from PEP 553
Message-ID: <5da71191.1c69fb81.6cbc5.1ec7@mx.google.com>

Author: Yannick Jadoul <yannick.jadoul at belgacom.net>
Branch: py3.7-pep553
Changeset: r97791:075c2edf30ee
Date: 2019-10-12 23:44 +0200
http://bitbucket.org/pypy/pypy/changeset/075c2edf30ee/

Log:	Implemented breakpoint and sys.breakpointhook from PEP 553

diff --git a/pypy/module/__builtin__/app_breakpoint.py b/pypy/module/__builtin__/app_breakpoint.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__builtin__/app_breakpoint.py
@@ -0,0 +1,16 @@
+# NOT_RPYTHON
+"""
+Plain Python definition of the builtin breakpoint function.
+"""
+
+import sys
+
+def breakpoint(*args, **kwargs):
+    """Call sys.breakpointhook(*args, **kws).  sys.breakpointhook() must accept
+whatever arguments are passed.
+
+By default, this drops you into the pdb debugger."""
+
+    if not hasattr(sys, 'breakpointhook'):
+        raise RuntimeError('lost sys.breakpointhook')
+    return sys.breakpointhook(*args, **kwargs)
diff --git a/pypy/module/__builtin__/moduledef.py b/pypy/module/__builtin__/moduledef.py
--- a/pypy/module/__builtin__/moduledef.py
+++ b/pypy/module/__builtin__/moduledef.py
@@ -23,6 +23,8 @@
         'bin'           : 'app_operation.bin',
         'oct'           : 'app_operation.oct',
         'hex'           : 'app_operation.hex',
+
+        'breakpoint'    : 'app_breakpoint.breakpoint',
     }
 
     interpleveldefs = {
diff --git a/pypy/module/sys/app.py b/pypy/module/sys/app.py
--- a/pypy/module/sys/app.py
+++ b/pypy/module/sys/app.py
@@ -68,6 +68,31 @@
     except:
         return False    # got an exception again... ignore, report the original
 
+def breakpointhook(*args, **kwargs):
+    """This hook function is called by built-in breakpoint()."""
+
+    import importlib, os, warnings
+
+    hookname = os.getenv('PYTHONBREAKPOINT')
+    if hookname is None or len(hookname) == 0:
+        hookname = 'pdb.set_trace'
+    elif hookname == '0':
+        return None
+    modname, dot, funcname = hookname.rpartition('.')
+    if dot == '':
+        modname = 'builtins'
+
+    try:
+        module = importlib.import_module(modname)
+        hook = getattr(module, funcname)
+    except:
+        warnings.warn(
+            'Ignoring unimportable $PYTHONBREAKPOINT: "{}"'.format(hookname),
+            RuntimeWarning)
+        return None
+
+    return hook(*args, **kwargs)
+
 def exit(exitcode=None):
     """Exit the interpreter by raising SystemExit(exitcode).
 If the exitcode is omitted or None, it defaults to zero (i.e., success).
diff --git a/pypy/module/sys/moduledef.py b/pypy/module/sys/moduledef.py
--- a/pypy/module/sys/moduledef.py
+++ b/pypy/module/sys/moduledef.py
@@ -109,6 +109,8 @@
     appleveldefs = {
         'excepthook'            : 'app.excepthook',
         '__excepthook__'        : 'app.excepthook',
+        'breakpointhook'        : 'app.breakpointhook',
+        '__breakpointhook__'    : 'app.breakpointhook',
         'exit'                  : 'app.exit',
         'callstats'             : 'app.callstats',
         'copyright'             : 'app.copyright_str',

From pypy.commits at gmail.com  Wed Oct 16 08:48:19 2019
From: pypy.commits at gmail.com (Yannick_Jadoul)
Date: Wed, 16 Oct 2019 05:48:19 -0700 (PDT)
Subject: [pypy-commit] pypy py3.7-pep553: Adding tests for breakpoint() and
 sys.breakpointhook()
Message-ID: <5da71193.1c69fb81.3f0ac.e8d6@mx.google.com>

Author: Yannick Jadoul <yannick.jadoul at belgacom.net>
Branch: py3.7-pep553
Changeset: r97792:085278e256df
Date: 2019-10-15 19:53 +0200
http://bitbucket.org/pypy/pypy/changeset/085278e256df/

Log:	Adding tests for breakpoint() and sys.breakpointhook()

diff --git a/pypy/module/__builtin__/test/test_breakpoint.py b/pypy/module/__builtin__/test/test_breakpoint.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__builtin__/test/test_breakpoint.py
@@ -0,0 +1,158 @@
+class AppTestBreakpoint:
+    def setup_class(cls):
+        cls.w_import_mock_pdb = cls.space.appexec([], """():
+            import sys
+            from contextlib import contextmanager
+            from types import ModuleType
+
+            @contextmanager
+            def w_import_mock_pdb():
+                try:
+                    mock_pdb = ModuleType('pdb')
+                    mock_pdb.set_trace = None
+                    sys.modules['pdb'] = mock_pdb
+                    yield mock_pdb
+                finally:
+                    del sys.modules['pdb']
+
+            return w_import_mock_pdb
+        """)
+
+        cls.w_mock_function = cls.space.appexec([], """():
+            from contextlib import contextmanager
+
+            class MockedCallable:
+                def __init__(self):
+                    self.called = 0
+
+                def __call__(self, *args, **kwargs):
+                    self.called += 1
+                    self.last_call = (args, kwargs)
+
+            @contextmanager
+            def mock_function(scope, attr, delete=False):
+                old = getattr(scope, attr)
+                try:
+                    if not delete:
+                        new = MockedCallable()
+                        setattr(scope, attr, new)
+                        yield new
+                    else:
+                        delattr(scope, attr)
+                        yield
+                finally:
+                    setattr(scope, attr, old)
+
+            return mock_function
+        """)
+
+
+    def test_default(self):
+        import sys
+        assert sys.breakpointhook is sys.__breakpointhook__
+
+        import os
+        assert 'PYTHONBREAKPOINT' not in os.environ
+
+        with self.import_mock_pdb() as pdb:
+            with self.mock_function(pdb, 'set_trace') as mocked:
+                breakpoint()
+                assert mocked.called == 1
+
+
+    def test_args_kwargs(self):
+        import sys
+        with self.import_mock_pdb() as pdb:
+            with self.mock_function(pdb, 'set_trace') as mocked:
+                breakpoint(1, 2, 3, x=4, y=5)
+                assert mocked.called == 1
+                assert mocked.last_call == ((1, 2, 3), {'x': 4, 'y': 5})
+
+
+    def test_breakpointhook(self):
+        import sys
+        with self.import_mock_pdb() as pdb:
+            with self.mock_function(sys, 'breakpointhook') as mocked, \
+                self.mock_function(pdb, 'set_trace') as mocked_pdb:
+                breakpoint()
+                assert mocked.called == 1
+                assert mocked_pdb.called == 0
+
+            with self.mock_function(pdb, 'set_trace') as mocked_pdb:
+                breakpoint()
+                assert mocked_pdb.called == 1
+
+
+    def test_breakpointhook_lost(self):
+        import sys
+        with self.import_mock_pdb() as pdb:
+            with self.mock_function(sys, 'breakpointhook', delete=True):
+                with raises(RuntimeError) as excinfo:
+                    breakpoint()
+                assert str(excinfo.value) == "lost sys.breakpointhook"
+
+
+    def test_env_default(self):
+        import os
+        try:
+            os.environ['PYTHONBREAKPOINT'] = ""
+
+            import sys
+            with self.import_mock_pdb() as pdb:
+                with self.mock_function(pdb, 'set_trace') as mocked:
+                    breakpoint()
+                    assert mocked.called == 1
+        finally:
+            del os.environ['PYTHONBREAKPOINT']
+
+
+    def test_env_disable(self):
+        import os
+        try:
+            os.environ['PYTHONBREAKPOINT'] = "0"
+
+            import sys
+            with self.import_mock_pdb() as pdb:
+                with self.mock_function(pdb, 'set_trace') as mocked:
+                    breakpoint()
+                    assert mocked.called == 0
+
+                with self.mock_function(sys, 'breakpointhook') as mocked:
+                    breakpoint()
+                    assert mocked.called == 1
+        finally:
+            del os.environ['PYTHONBREAKPOINT']
+
+
+    def test_env_other(self):
+        import os
+        try:
+            os.environ['PYTHONBREAKPOINT'] = 'sys.exit'
+
+            import sys
+            with self.import_mock_pdb() as pdb:
+                with self.mock_function(sys, 'exit') as mocked, \
+                     self.mock_function(pdb, 'set_trace') as mocked_pdb:
+                    breakpoint()
+                    assert mocked.called == 1
+                    assert mocked_pdb.called == 0
+        finally:
+            del os.environ['PYTHONBREAKPOINT']
+
+
+    def test_env_nonexistent(self):
+        import os
+        import warnings
+        try:
+            os.environ['PYTHONBREAKPOINT'] = 'blah.bleh'
+
+            import sys
+            with self.import_mock_pdb() as pdb:
+                with self.mock_function(pdb, 'set_trace') as mocked_pdb, \
+                     warnings.catch_warnings(record=True) as w:
+                    breakpoint()
+                    assert mocked_pdb.called == 0
+                    assert len(w) == 1
+                    assert str(w[-1].message) == 'Ignoring unimportable $PYTHONBREAKPOINT: "blah.bleh"'
+        finally:
+            del os.environ['PYTHONBREAKPOINT']

From pypy.commits at gmail.com  Wed Oct 16 09:07:42 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 16 Oct 2019 06:07:42 -0700 (PDT)
Subject: [pypy-commit] pypy record-known-result: two more tests the branch
 makes possible
Message-ID: <5da7161e.1c69fb81.56c8.e262@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: record-known-result
Changeset: r97794:1bdeb2170bbc
Date: 2019-10-16 15:06 +0200
http://bitbucket.org/pypy/pypy/changeset/1bdeb2170bbc/

Log:	two more tests the branch makes possible

diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -271,7 +271,6 @@
         def main(n):
             b = b'ab\xc3\xa4\xf0\x9f\x91\xa9\xe2\x80\x8d\xf0\x9f\x91\xa9\xe2\x80\x8d\xf0\x9f\x91\xa7\xe2\x80\x8d\xf0\x9f\x91\xa6'
             u = b.decode("utf-8") * 1000
-            global s
             count = 0
             while u:
                 u = u[1:] # ID: index
@@ -293,7 +292,6 @@
     def test_decode_encode(self):
         log = self.run("""
         def main(n):
-            global s
             u = b'ab\xc3\xa4\xf0\x9f\x91\xa9\xe2\x80\x8d\xf0\x9f\x91\xa9\xe2\x80\x8d\xf0\x9f\x91\xa7\xe2\x80\x8d\xf0\x9f\x91\xa6'.decode("utf-8")
             count = 0
             for i in range(n):
@@ -309,3 +307,68 @@
             i95 = int_ge(i86, 0)
             guard_true(i95, descr=...)
         ''')
+
+    def test_access_two_indices_in_a_row(self):
+        log = self.run("""
+        def main(n):
+            u = b'ab\xc3\xa4\xf0\x9f\x91\xa9\xe2\x80\x8d\xf0\x9f\x91\xa9\xe2\x80\x8d\xf0\x9f\x91\xa7\xe2\x80\x8d\xf0\x9f\x91\xa6'.decode("utf-8")
+            u *= 100
+            for i in range(len(u) - 1):
+                u[i]
+                u[i + 1] # ID: second
+        """, [10000])
+        loop, = log.loops_by_filename(self.filepath)
+        # There is no _codepoint_position_at_index call here! It's deduced as
+        # the result of the next_codepoint_pos of the previous indexing.
+        assert loop.match_by_id('second', '''
+            i80 = int_ge(i70, i39)
+            guard_false(i80, descr=...)
+            i82 = call_i(ConstClass(next_codepoint_pos_dont_look_inside), p40, i77, descr=...)
+            i84 = int_add(i65, 2)
+            i85 = int_sub(i82, i77)
+            guard_not_invalidated(descr=...)
+            --TICK--
+        ''')
+
+    def test_find_dont_convert_back_and_forth(self):
+        log = self.run("""
+        def main(n):
+            u = b'ab\xc3\xa4\xf0\x9f\x91\xa9\xe2\x80\x8d\xf0\x9f\x91\xa9\xe2\x80\x8d\xf0\x9f\x91\xa7\xe2\x80\x8d\xf0\x9f\x91\xa6a'.decode("utf-8")
+            u *= 100
+            for i in range(len(u) - 1):
+                u[u.find(u"a", i)]  # ID: find
+        """, [10000])
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match_by_id('find', '''
+            setfield_gc(p16, i91, descr=...)
+            guard_not_invalidated(descr=...)
+            i92 = int_is_zero(i86)
+            guard_false(i92, descr=...)
+            i93 = int_gt(i86, i44)
+            guard_false(i93, descr=...)
+            p95 = cond_call_value_r(p66, ConstClass(W_UnicodeObject._compute_index_storage), p12, descr=...)
+            guard_no_exception(descr=...)
+            i97 = call_i(ConstClass(_codepoint_position_at_index), p47, p95, i86, descr=...)
+            guard_no_exception(descr=...)
+            i99 = int_lt(i97, 0)
+            guard_false(i99, descr=...)
+            i100 = int_sub(i48, i97)
+            i102 = int_lt(i100, 0)
+            guard_false(i102, descr=...)
+            i105 = call_i(ConstClass(ll_find_char__rpy_stringPtr_Char_Signed_Signed), p47, 97, i97, i48, descr=...)
+            i107 = int_lt(i105, 0)
+            guard_false(i107, descr=...)
+            p108 = getfield_gc_r(p12, descr=...)
+
+            # This call converts the byte position that is the result of the
+            # find call back to a codepoint position.
+            i110 = call_i(ConstClass(_codepoint_index_at_byte_position), p47, p95, i105, i44, descr=...)
+
+            # There is no call to codepoint_position_at_index here! It's
+            # deduced as the reverse of _codepoint_index_at_byte_position.
+            guard_no_exception(descr=...)
+            i112 = call_i(ConstClass(next_codepoint_pos_dont_look_inside), p47, i105, descr=...)
+            i114 = int_add(i110, 1)
+            i115 = int_sub(i112, i105)
+            --TICK--
+        ''')

From pypy.commits at gmail.com  Wed Oct 16 11:20:39 2019
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 16 Oct 2019 08:20:39 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6-asyncgen: Close branch py3.6-asyncgen
Message-ID: <5da73547.1c69fb81.a4e43.1669@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6-asyncgen
Changeset: r97795:2ca58cb34e7f
Date: 2019-10-16 15:19 +0000
http://bitbucket.org/pypy/pypy/changeset/2ca58cb34e7f/

Log:	Close branch py3.6-asyncgen


From pypy.commits at gmail.com  Wed Oct 16 11:21:32 2019
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 16 Oct 2019 08:21:32 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: Merged in py3.6-asyncgen (pull request
 #672)
Message-ID: <5da7357c.1c69fb81.f8e83.e019@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6
Changeset: r97796:2ccaa27c8549
Date: 2019-10-16 15:19 +0000
http://bitbucket.org/pypy/pypy/changeset/2ccaa27c8549/

Log:	Merged in py3.6-asyncgen (pull request #672)

	Fix asyncgen_hooks and refactor coroutine execution

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1234,8 +1234,11 @@
         from pypy.interpreter.generator import GeneratorIterator
         return isinstance(w_obj, GeneratorIterator)
 
+    def callable_w(self, w_obj):
+        return self.lookup(w_obj, "__call__") is not None
+
     def callable(self, w_obj):
-        return self.newbool(self.lookup(w_obj, "__call__") is not None)
+        return self.newbool(self.callable_w(w_obj))
 
     def issequence_w(self, w_obj):
         flag = self.type(w_obj).flag_map_or_seq
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -23,8 +23,9 @@
     # XXX [fijal] but they're not. is_being_profiled is guarded a bit all
     #     over the place as well as w_tracefunc
 
-    _immutable_fields_ = ['profilefunc?', 'w_tracefunc?',
-                          'w_coroutine_wrapper_fn?']
+    _immutable_fields_ = [
+        'profilefunc?', 'w_tracefunc?', 'w_coroutine_wrapper_fn?',
+        'w_asyncgen_firstiter_fn?', 'w_asyncgen_finalizer_fn?']
 
     def __init__(self, space):
         self.space = space
@@ -41,6 +42,8 @@
         self.thread_disappeared = False   # might be set to True after os.fork()
         self.w_coroutine_wrapper_fn = None
         self.in_coroutine_wrapper = False
+        self.w_asyncgen_firstiter_fn = None
+        self.w_asyncgen_finalizer_fn = None
 
     @staticmethod
     def _mark_thread_disappeared(space):
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -11,8 +11,6 @@
 class GeneratorOrCoroutine(W_Root):
     _immutable_fields_ = ['pycode']
 
-    w_yielded_from = None
-
     def __init__(self, frame, name=None, qualname=None):
         self.space = frame.space
         self.frame = frame     # turned into None when frame_finished_execution
@@ -84,7 +82,7 @@
                 operr = OperationError(space.w_StopIteration, space.w_None)
             raise operr
 
-        w_result = self._invoke_execute_frame(frame, w_arg_or_err)
+        w_result = self._invoke_execute_frame(w_arg_or_err)
         assert w_result is not None
 
         # if the frame is now marked as finished, it was RETURNed from
@@ -102,8 +100,9 @@
         else:
             return w_result     # YIELDed
 
-    def _invoke_execute_frame(self, frame, w_arg_or_err):
+    def _invoke_execute_frame(self, w_arg_or_err):
         space = self.space
+        frame = self.frame
         if self.running:
             raise oefmt(space.w_ValueError, "%s already executing", self.KIND)
         ec = space.getexecutioncontext()
@@ -126,7 +125,7 @@
                             self.KIND)
         self.running = True
         try:
-            w_result = frame.execute_frame(self, w_arg_or_err)
+            w_result = frame.execute_frame(w_arg_or_err)
         except OperationError as e:
             # errors finish a frame
             try:
@@ -149,68 +148,19 @@
             ec.set_sys_exc_info(current_exc_info)
         return w_result
 
-    def resume_execute_frame(self, frame, w_arg_or_err):
-        # Called from execute_frame() just before resuming the bytecode
-        # interpretation.
-        space = self.space
-        w_yf = self.w_yielded_from
-        if w_yf is not None:
-            self.w_yielded_from = None
-            try:
-                self.next_yield_from(frame, w_yf, w_arg_or_err)
-            except OperationError as operr:
-                operr.record_context(space, space.getexecutioncontext())
-                return frame.handle_generator_error(operr)
-            # Normal case: the call above raises Yield.
-            # We reach this point if the iterable is exhausted.
-            last_instr = jit.promote(frame.last_instr)
-            assert last_instr & 1 == 0
-            assert last_instr >= 0
-            return r_uint(last_instr + 2)
+    def get_delegate(self):
+        if self.frame is None:
+            return None
+        return self.frame.w_yielding_from
 
-        if isinstance(w_arg_or_err, SApplicationException):
-            return frame.handle_generator_error(w_arg_or_err.operr)
+    def descr_delegate(self, space):
+        w_yf = self.get_delegate()
+        if w_yf is None:
+            return space.w_None
+        return w_yf
 
-        last_instr = jit.promote(frame.last_instr)
-        if last_instr != -1:
-            assert last_instr & 1 == 0
-            frame.pushvalue(w_arg_or_err)
-            return r_uint(last_instr + 2)
-        else:
-            return r_uint(0)
-
-    def next_yield_from(self, frame, w_yf, w_inputvalue_or_err):
-        """Fetch the next item of the current 'yield from', push it on
-        the frame stack, and raises Yield.  If there isn't one, push
-        w_stopiteration_value and returns.  May also just raise.
-        """
-        space = self.space
-        try:
-            if isinstance(w_yf, GeneratorOrCoroutine):
-                w_retval = w_yf.send_ex(w_inputvalue_or_err)
-            elif isinstance(w_yf, AsyncGenASend):   # performance only
-                w_retval = w_yf.do_send(w_inputvalue_or_err)
-            elif space.is_w(w_inputvalue_or_err, space.w_None):
-                w_retval = space.next(w_yf)
-            else:
-                w_retval = delegate_to_nongen(space, w_yf, w_inputvalue_or_err)
-        except OperationError as e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            frame._report_stopiteration_sometimes(w_yf, e)
-            try:
-                w_stop_value = space.getattr(e.get_w_value(space),
-                                             space.newtext("value"))
-            except OperationError as e:
-                if not e.match(space, space.w_AttributeError):
-                    raise
-                w_stop_value = space.w_None
-            frame.pushvalue(w_stop_value)
-            return
-        else:
-            frame.pushvalue(w_retval)
-            self.w_yielded_from = w_yf
-            raise Yield
+    def set_delegate(self, w_delegate):
+        self.frame.w_yielding_from = w_delegate
 
     def _leak_stopiteration(self, e):
         # Check for __future__ generator_stop and conditionally turn
@@ -259,8 +209,8 @@
         operr = OperationError(w_type, w_val, tb)
         operr.normalize_exception(space)
 
-        # note: w_yielded_from is always None if 'self.running'
-        if (self.w_yielded_from is not None and
+        # note: _w_yielded_from is always None if 'self.running'
+        if (self.get_delegate() is not None and
                     operr.match(space, space.w_GeneratorExit)):
             try:
                 self._gen_close_iter(space)
@@ -276,8 +226,8 @@
 
     def _gen_close_iter(self, space):
         assert not self.running
-        w_yf = self.w_yielded_from
-        self.w_yielded_from = None
+        w_yf = self.get_delegate()
+        self.set_delegate(None)
         self.running = True
         try:
             gen_close_iter(space, w_yf)
@@ -290,8 +240,8 @@
             return     # nothing to do in this case
         space = self.space
         operr = None
-        # note: w_yielded_from is always None if 'self.running'
-        w_yf = self.w_yielded_from
+        # note: _w_yielded_from is always None if 'self.running'
+        w_yf = self.get_delegate()
         if w_yf is not None:
             try:
                 self._gen_close_iter(space)
@@ -375,7 +325,7 @@
     # generate 2 versions of the function and 2 jit drivers.
     def _create_unpack_into():
         jitdriver = jit.JitDriver(greens=['pycode'],
-                                  reds=['self', 'frame', 'results'],
+                                  reds='auto',
                                   name='unpack_into')
 
         def unpack_into(self, results):
@@ -386,12 +336,10 @@
                 return
             pycode = self.pycode
             while True:
-                jitdriver.jit_merge_point(self=self, frame=frame,
-                                          results=results, pycode=pycode)
+                jitdriver.jit_merge_point(pycode=pycode)
                 space = self.space
                 try:
-                    w_result = self._invoke_execute_frame(
-                                            frame, space.w_None)
+                    w_result = self._invoke_execute_frame(space.w_None)
                 except OperationError as e:
                     if not e.match(space, space.w_StopIteration):
                         raise
@@ -489,27 +437,6 @@
         else:
             space.call_function(w_close)
 
-def delegate_to_nongen(space, w_yf, w_inputvalue_or_err):
-    # invoke a "send" or "throw" by method name to a non-generator w_yf
-    if isinstance(w_inputvalue_or_err, SApplicationException):
-        operr = w_inputvalue_or_err.operr
-        try:
-            w_meth = space.getattr(w_yf, space.newtext("throw"))
-        except OperationError as e:
-            if not e.match(space, space.w_AttributeError):
-                raise
-            raise operr
-        # bah, CPython calls here with the exact same arguments as
-        # originally passed to throw().  In our case it is far removed.
-        # Let's hope nobody will complain...
-        operr.normalize_exception(space)
-        w_exc = operr.w_type
-        w_val = operr.get_w_value(space)
-        w_tb  = operr.get_w_traceback(space)
-        return space.call_function(w_meth, w_exc, w_val, w_tb)
-    else:
-        return space.call_method(w_yf, "send", w_inputvalue_or_err)
-
 def gen_is_coroutine(w_obj):
     return (isinstance(w_obj, GeneratorIterator) and
             (w_obj.pycode.co_flags & consts.CO_ITERABLE_COROUTINE) != 0)
@@ -594,15 +521,15 @@
         if self.hooks_inited:
             return
         self.hooks_inited = True
-
-        self.w_finalizer = self.space.appexec([], '''():
-            import sys
-            hooks = sys.get_asyncgen_hooks()
-            return hooks.finalizer''')
+        ec = self.space.getexecutioncontext()
+        self.w_finalizer = ec.w_asyncgen_finalizer_fn
+        w_firstiter = ec.w_asyncgen_firstiter_fn
+        if w_firstiter is not None:
+            self.space.call_function(w_firstiter, self)
 
     def _finalize_(self):
         if self.frame is not None and self.frame.lastblock is not None:
-            if self.w_finalizer is not self.space.w_None:
+            if self.w_finalizer is not None:
                 # XXX: this is a hack to resurrect the weakref that was cleared
                 # before running _finalize_()
                 if self.space.config.translation.rweakref:
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -68,6 +68,7 @@
     frame_finished_execution = False
     f_generator_wref         = rweakref.dead_ref  # for generators/coroutines
     f_generator_nowref       = None               # (only one of the two attrs)
+    w_yielding_from = None
     last_instr               = -1
     f_backref                = jit.vref_None
 
@@ -319,9 +320,40 @@
                 ec.in_coroutine_wrapper = False
         return w_gen
 
-    def execute_frame(self, in_generator=None, w_arg_or_err=None):
+    def resume_execute_frame(self, w_arg_or_err):
+        # Called from execute_frame() just before resuming the bytecode
+        # interpretation.
+        from pypy.interpreter.pyopcode import SApplicationException
+        space = self.space
+        w_yf = self.w_yielding_from
+        if w_yf is not None:
+            self.w_yielding_from = None
+            try:
+                self.next_yield_from(w_yf, w_arg_or_err)
+            except OperationError as operr:
+                operr.record_context(space, space.getexecutioncontext())
+                return self.handle_generator_error(operr)
+            # Normal case: the call above raises Yield.
+            # We reach this point if the iterable is exhausted.
+            last_instr = jit.promote(self.last_instr)
+            assert last_instr & 1 == 0
+            assert last_instr >= 0
+            return r_uint(last_instr + 2)
+
+        if isinstance(w_arg_or_err, SApplicationException):
+            return self.handle_generator_error(w_arg_or_err.operr)
+
+        last_instr = jit.promote(self.last_instr)
+        if last_instr != -1:
+            assert last_instr & 1 == 0
+            self.pushvalue(w_arg_or_err)
+            return r_uint(last_instr + 2)
+        else:
+            return r_uint(0)
+
+    def execute_frame(self, w_arg_or_err=None):
         """Execute this frame.  Main entry point to the interpreter.
-        'in_generator' is non-None iff we are starting or resuming
+        'w_arg_or_err' is non-None iff we are starting or resuming
         a generator or coroutine frame; in that case, w_arg_or_err
         is the input argument -or- an SApplicationException instance.
         """
@@ -343,12 +375,11 @@
             # the YIELD_VALUE/YIELD_FROM instruction.
             try:
                 try:
-                    if in_generator is None:
+                    if w_arg_or_err is None:
                         assert self.last_instr == -1
                         next_instr = r_uint(0)
                     else:
-                        next_instr = in_generator.resume_execute_frame(
-                                                        self, w_arg_or_err)
+                        next_instr = self.resume_execute_frame(w_arg_or_err)
                 except pyopcode.Yield:
                     w_exitvalue = self.popvalue()
                 else:
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1113,24 +1113,51 @@
             self.pushvalue(w_value)
         raise Yield
 
+    def next_yield_from(self, w_yf, w_inputvalue_or_err):
+        """Fetch the next item of the current 'yield from', push it on
+        the frame stack, and raises Yield.  If there isn't one, push
+        w_stopiteration_value and returns.  May also just raise.
+        """
+        from pypy.interpreter.generator import (
+            GeneratorOrCoroutine, AsyncGenASend)
+        space = self.space
+        try:
+            if isinstance(w_yf, GeneratorOrCoroutine):
+                w_retval = w_yf.send_ex(w_inputvalue_or_err)
+            elif isinstance(w_yf, AsyncGenASend):   # performance only
+                w_retval = w_yf.do_send(w_inputvalue_or_err)
+            elif space.is_w(w_inputvalue_or_err, space.w_None):
+                w_retval = space.next(w_yf)
+            else:
+                w_retval = delegate_to_nongen(space, w_yf, w_inputvalue_or_err)
+        except OperationError as e:
+            if not e.match(space, space.w_StopIteration):
+                raise
+            self._report_stopiteration_sometimes(w_yf, e)
+            try:
+                w_stop_value = space.getattr(e.get_w_value(space),
+                                             space.newtext("value"))
+            except OperationError as e:
+                if not e.match(space, space.w_AttributeError):
+                    raise
+                w_stop_value = space.w_None
+            self.pushvalue(w_stop_value)
+            return
+        else:
+            self.pushvalue(w_retval)
+            self.w_yielding_from = w_yf
+            raise Yield
+
     def YIELD_FROM(self, oparg, next_instr):
         # Unlike CPython, we handle this not by repeating the same
         # bytecode over and over until the inner iterator is exhausted.
-        # Instead, we directly set the generator's w_yielded_from.
-        # This asks generator.resume_execute_frame() to exhaust that
+        # Instead, we set w_yielding_from.
+        # This asks resume_execute_frame() to exhaust that
         # sub-iterable first before continuing on the next bytecode.
-        in_generator = self.get_generator()
-        if in_generator is None:
-            # Issue #2768: rare case involving __del__ methods.
-            # XXX This is a workaround, not a proper solution.
-            raise oefmt(self.space.w_RuntimeError,
-                        "PyPy limitation: cannot use 'yield from' or 'await' "
-                        "in a generator/coroutine that has been partially "
-                        "deallocated already, typically seen via __del__")
         w_inputvalue = self.popvalue()    # that's always w_None, actually
         w_gen = self.popvalue()
         #
-        in_generator.next_yield_from(self, w_gen, w_inputvalue)
+        self.next_yield_from(w_gen, w_inputvalue)
         # Common case: the call above raises Yield.
         # If instead the iterable is empty, next_yield_from() pushed the
         # final result and returns.  In that case, we can just continue
@@ -1556,7 +1583,7 @@
         w_iterable = self.popvalue()
         w_iter = get_awaitable_iter(self.space, w_iterable)
         if isinstance(w_iter, Coroutine):
-            if w_iter.w_yielded_from is not None:
+            if w_iter.get_delegate() is not None:
                 # 'w_iter' is a coroutine object that is being awaited,
                 # '.w_yielded_from' is the current awaitable being awaited on.
                 raise oefmt(self.space.w_RuntimeError,
@@ -1694,6 +1721,27 @@
         else:
             self.MISSING_OPCODE(oparg, next_instr)
 
+def delegate_to_nongen(space, w_yf, w_inputvalue_or_err):
+    # invoke a "send" or "throw" by method name to a non-generator w_yf
+    if isinstance(w_inputvalue_or_err, SApplicationException):
+        operr = w_inputvalue_or_err.operr
+        try:
+            w_meth = space.getattr(w_yf, space.newtext("throw"))
+        except OperationError as e:
+            if not e.match(space, space.w_AttributeError):
+                raise
+            raise operr
+        # bah, CPython calls here with the exact same arguments as
+        # originally passed to throw().  In our case it is far removed.
+        # Let's hope nobody will complain...
+        operr.normalize_exception(space)
+        w_exc = operr.w_type
+        w_val = operr.get_w_value(space)
+        w_tb = operr.get_w_traceback(space)
+        return space.call_function(w_meth, w_exc, w_val, w_tb)
+    else:
+        return space.call_method(w_yf, "send", w_inputvalue_or_err)
+
 
 ### ____________________________________________________________ ###
 
diff --git a/pypy/interpreter/test/apptest_coroutine.py b/pypy/interpreter/test/apptest_coroutine.py
--- a/pypy/interpreter/test/apptest_coroutine.py
+++ b/pypy/interpreter/test/apptest_coroutine.py
@@ -633,16 +633,12 @@
         a2 = g.aclose()
     sys.set_asyncgen_hooks(finalizer=_finalize)
     assert state == 0
-    try:
+    with pytest.raises(StopIteration):
         a.send(None)
-    except StopIteration:
-        pass
     assert a2.send(None) == 'coro'
     assert state == 1
-    try:
+    with pytest.raises(StopIteration):
         a2.send(None)
-    except StopIteration:
-        pass
     assert state == 2
     sys.set_asyncgen_hooks(None, None)
 
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -838,7 +838,7 @@
     gi_running = interp_attrproperty('running', cls=GeneratorIterator, wrapfn="newbool"),
     gi_frame   = GetSetProperty(GeneratorIterator.descr_gicr_frame),
     gi_code    = interp_attrproperty_w('pycode', cls=GeneratorIterator),
-    gi_yieldfrom=interp_attrproperty_w('w_yielded_from', cls=GeneratorIterator),
+    gi_yieldfrom=GetSetProperty(GeneratorIterator.descr_delegate),
     __name__   = GetSetProperty(GeneratorIterator.descr__name__,
                                 GeneratorIterator.descr_set__name__),
     __qualname__ = GetSetProperty(GeneratorIterator.descr__qualname__,
@@ -862,7 +862,7 @@
     cr_running = interp_attrproperty('running', cls=Coroutine, wrapfn="newbool"),
     cr_frame   = GetSetProperty(Coroutine.descr_gicr_frame),
     cr_code    = interp_attrproperty_w('pycode', cls=Coroutine),
-    cr_await   = interp_attrproperty_w('w_yielded_from', cls=Coroutine),
+    cr_await=GetSetProperty(Coroutine.descr_delegate),
     __name__   = GetSetProperty(Coroutine.descr__name__,
                                 Coroutine.descr_set__name__,
                                 doc="name of the coroutine"),
@@ -890,7 +890,7 @@
     ag_running = interp_attrproperty('running', cls=AsyncGenerator, wrapfn="newbool"),
     ag_frame   = GetSetProperty(AsyncGenerator.descr_gicr_frame),
     ag_code    = interp_attrproperty_w('pycode', cls=AsyncGenerator),
-    ag_await   = interp_attrproperty_w('w_yielded_from', cls=AsyncGenerator),
+    ag_await=GetSetProperty(AsyncGenerator.descr_delegate),
     __name__   = GetSetProperty(AsyncGenerator.descr__name__,
                                 AsyncGenerator.descr_set__name__,
                                 doc="name of the async generator"),
diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -9,15 +9,15 @@
 # ____________________________________________________________
 
 
-_get_code = lambda frame, w_inputvalue, operr: frame.pycode
+_get_code = lambda frame, w_arg_or_err: frame.pycode
 _decorator = rvmprof.vmprof_execute_code("pypy", _get_code, W_Root)
 my_execute_frame = _decorator(PyFrame.execute_frame)
 
 
 class __extend__(PyFrame):
-    def execute_frame(self, in_generator=None, w_arg_or_err=None):
+    def execute_frame(self, w_arg_or_err=None):
         # indirection for the optional arguments
-        return my_execute_frame(self, in_generator, w_arg_or_err)
+        return my_execute_frame(self, w_arg_or_err)
 
 
 def _safe(s):
diff --git a/pypy/module/sys/app.py b/pypy/module/sys/app.py
--- a/pypy/module/sys/app.py
+++ b/pypy/module/sys/app.py
@@ -75,7 +75,7 @@
 If it is another kind of object, it will be printed and the system
 exit status will be one (i.e., failure)."""
     # note that we cannot simply use SystemExit(exitcode) here.
-    # in the default branch, we use "raise SystemExit, exitcode", 
+    # in the default branch, we use "raise SystemExit, exitcode",
     # which leads to an extra de-tupelizing
     # in normalize_exception, which is exactly like CPython's.
     if isinstance(exitcode, tuple):
@@ -109,7 +109,6 @@
 
 # This is tested in test_app_main.py
 class sysflags(metaclass=structseqtype):
-
     name = "sys.flags"
 
     debug = structseqfield(0)
@@ -130,29 +129,6 @@
 null__xoptions = {}
 
 
-class asyncgen_hooks(metaclass=structseqtype):
-    name = "asyncgen_hooks"
-
-    firstiter = structseqfield(0)
-    finalizer = structseqfield(1)
-
-# FIXME: make this thread-local
-_current_asyncgen_hooks = asyncgen_hooks((None, None))
-
-def get_asyncgen_hooks():
-    return _current_asyncgen_hooks
-
-_default_arg = object()
-
-def set_asyncgen_hooks(firstiter=_default_arg, finalizer=_default_arg):
-    global _current_asyncgen_hooks
-    if firstiter is _default_arg:
-        firstiter = _current_asyncgen_hooks.firstiter
-    if finalizer is _default_arg:
-        finalizer = _current_asyncgen_hooks.finalizer
-    _current_asyncgen_hooks = asyncgen_hooks((firstiter, finalizer))
-
-
 implementation = SimpleNamespace(
     name='pypy',
     version=sys.version_info,
diff --git a/pypy/module/sys/moduledef.py b/pypy/module/sys/moduledef.py
--- a/pypy/module/sys/moduledef.py
+++ b/pypy/module/sys/moduledef.py
@@ -95,6 +95,8 @@
 
         'get_coroutine_wrapper' : 'vm.get_coroutine_wrapper',
         'set_coroutine_wrapper' : 'vm.set_coroutine_wrapper',
+        'get_asyncgen_hooks'    : 'vm.get_asyncgen_hooks',
+        'set_asyncgen_hooks'    : 'vm.set_asyncgen_hooks',
 
         'is_finalizing'         : 'vm.is_finalizing',
         }
@@ -115,8 +117,6 @@
         'flags'                 : 'app.null_sysflags',
         '_xoptions'             : 'app.null__xoptions',
         'implementation'        : 'app.implementation',
-        'get_asyncgen_hooks'    : 'app.get_asyncgen_hooks',
-        'set_asyncgen_hooks'    : 'app.set_asyncgen_hooks',
 
         # these six attributes are here only during tests;
         # they are removed before translation
@@ -184,7 +184,7 @@
                     if w_file is w_stdout:
                         e.write_unraisable(space, '', w_file)
                     ret = -1
-        return ret 
+        return ret
 
     def _file_is_closed(self, space, w_file):
         try:
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -7,7 +7,7 @@
 
 from pypy.interpreter import gateway
 from pypy.interpreter.error import oefmt
-from pypy.interpreter.gateway import unwrap_spec
+from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
 
 
 # ____________________________________________________________
@@ -240,6 +240,14 @@
     suite_mask = structseqfield(12, "Bit mask identifying available product suites")
     product_type = structseqfield(13, "System product type")
     platform_version = structseqfield(14, "Diagnostic version number")
+
+
+class asyncgen_hooks(metaclass=structseqtype):
+    name = "asyncgen_hooks"
+
+    firstiter = structseqfield(0)
+    finalizer = structseqfield(1)
+
 ''')
 
 
@@ -286,7 +294,7 @@
 
 getsizeof_missing = """getsizeof(...)
     getsizeof(object, default) -> int
-    
+
     Return the size of object in bytes.
 
 sys.getsizeof(object, default) will always return default on PyPy, and
@@ -351,5 +359,45 @@
     else:
         raise oefmt(space.w_TypeError, "callable expected, got %T", w_wrapper)
 
+def get_asyncgen_hooks(space):
+    """get_asyncgen_hooks()
+
+Return a namedtuple of installed asynchronous generators hooks (firstiter, finalizer)."""
+    ec = space.getexecutioncontext()
+    w_firstiter = ec.w_asyncgen_firstiter_fn
+    if w_firstiter is None:
+        w_firstiter = space.w_None
+    w_finalizer = ec.w_asyncgen_finalizer_fn
+    if w_finalizer is None:
+        w_finalizer = space.w_None
+    w_asyncgen_hooks = app.wget(space, "asyncgen_hooks")
+    return space.call_function(
+        w_asyncgen_hooks,
+        space.newtuple([w_firstiter, w_finalizer]))
+
+# Note: the docstring is wrong on CPython
+def set_asyncgen_hooks(space, w_firstiter=None, w_finalizer=None):
+    """set_asyncgen_hooks(firstiter=None, finalizer=None)
+
+Set a finalizer for async generators objects."""
+    ec = space.getexecutioncontext()
+    if space.is_w(w_finalizer, space.w_None):
+        ec.w_asyncgen_finalizer_fn = None
+    elif w_finalizer is not None:
+        if space.callable_w(w_finalizer):
+            ec.w_asyncgen_finalizer_fn = w_finalizer
+        else:
+            raise oefmt(space.w_TypeError,
+                "callable finalizer expected, got %T", w_finalizer)
+    if space.is_w(w_firstiter, space.w_None):
+        ec.w_asyncgen_firstiter_fn = None
+    elif w_firstiter is not None:
+        if space.callable_w(w_firstiter):
+            ec.w_asyncgen_firstiter_fn = w_firstiter
+        else:
+            raise oefmt(space.w_TypeError,
+                "callable firstiter expected, got %T", w_firstiter)
+
+
 def is_finalizing(space):
     return space.newbool(space.sys.finalizing)

From pypy.commits at gmail.com  Wed Oct 16 11:49:21 2019
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 16 Oct 2019 08:49:21 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: Fix range checking in GB18030 decoder
 (bpo-29990)
Message-ID: <5da73c01.1c69fb81.e5c8a.f5a8@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6
Changeset: r97797:08277358d8fb
Date: 2019-10-16 16:48 +0100
http://bitbucket.org/pypy/pypy/changeset/08277358d8fb/

Log:	Fix range checking in GB18030 decoder (bpo-29990)

diff --git a/pypy/module/_multibytecodec/src/cjkcodecs/_codecs_cn.c b/pypy/module/_multibytecodec/src/cjkcodecs/_codecs_cn.c
--- a/pypy/module/_multibytecodec/src/cjkcodecs/_codecs_cn.c
+++ b/pypy/module/_multibytecodec/src/cjkcodecs/_codecs_cn.c
@@ -266,7 +266,9 @@
             REQUIRE_INBUF(4)
             c3 = IN3;
             c4 = IN4;
-            if (c < 0x81 || c3 < 0x81 || c4 < 0x30 || c4 > 0x39)
+            if (c  < 0x81 || c  > 0xFE ||
+                c3 < 0x81 || c3 > 0xFE ||
+                c4 < 0x30 || c4 > 0x39)
                 return 1;
             c -= 0x81;  c2 -= 0x30;
             c3 -= 0x81; c4 -= 0x30;
diff --git a/pypy/module/_multibytecodec/test/test_c_codecs.py b/pypy/module/_multibytecodec/test/test_c_codecs.py
--- a/pypy/module/_multibytecodec/test/test_c_codecs.py
+++ b/pypy/module/_multibytecodec/test/test_c_codecs.py
@@ -1,4 +1,5 @@
 import py
+import pytest
 from pypy.module._multibytecodec.c_codecs import getcodec, codecs
 from pypy.module._multibytecodec.c_codecs import decode, encode
 from pypy.module._multibytecodec.c_codecs import EncodeDecodeError
@@ -18,6 +19,15 @@
     u = decode(c, "foobar")
     assert u == "foobar"
 
+ at pytest.mark.parametrize('undecodable', [
+    b"abc\x80\x80\xc1\xc4",
+    b"\xff\x30\x81\x30", b"\x81\x30\xff\x30",  # bpo-29990
+])
+def test_decode_gb18030_error(undecodable):
+    c = getcodec("gb18030")
+    with pytest.raises(EncodeDecodeError):
+        decode(c, undecodable)
+
 def test_decode_hz():
     # stateful
     c = getcodec("hz")

From pypy.commits at gmail.com  Wed Oct 16 12:58:55 2019
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 16 Oct 2019 09:58:55 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: Fix handling escape characters in HZ
 codec (bpo-30003)
Message-ID: <5da74c4f.1c69fb81.3699c.6636@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6
Changeset: r97798:1c5e6646df47
Date: 2019-10-16 17:57 +0100
http://bitbucket.org/pypy/pypy/changeset/1c5e6646df47/

Log:	Fix handling escape characters in HZ codec (bpo-30003)

diff --git a/pypy/module/_multibytecodec/src/cjkcodecs/_codecs_cn.c b/pypy/module/_multibytecodec/src/cjkcodecs/_codecs_cn.c
--- a/pypy/module/_multibytecodec/src/cjkcodecs/_codecs_cn.c
+++ b/pypy/module/_multibytecodec/src/cjkcodecs/_codecs_cn.c
@@ -335,14 +335,16 @@
         DBCHAR code;
 
         if (c < 0x80) {
-            if (state->i == 0) {
-                WRITE1((unsigned char)c)
-                NEXT(1, 1)
+            if (state->i) {
+                WRITE2('~', '}');
+                NEXT_OUT(2);
+                state->i = 0;
             }
-            else {
-                WRITE3('~', '}', (unsigned char)c)
-                NEXT(1, 3)
-                state->i = 0;
+            WRITE1((unsigned char)c);
+            NEXT(1, 1);
+            if (c == '~') {
+                WRITE1('~');
+                NEXT_OUT(1);
             }
             continue;
         }
@@ -390,17 +392,15 @@
             unsigned char c2 = IN2;
 
             REQUIRE_INBUF(2)
-            if (c2 == '~') {
+            if (c2 == '~' && state->i == 0) {
                 WRITE1('~')
-                NEXT(2, 1)
-                continue;
             }
             else if (c2 == '{' && state->i == 0)
                 state->i = 1; /* set GB */
+            else if (c2 == '\n' && state->i == 0)
+                ; /* line-continuation */
             else if (c2 == '}' && state->i == 1)
                 state->i = 0; /* set ASCII */
-            else if (c2 == '\n')
-                ; /* line-continuation */
             else
                 return 1;
             NEXT(2, 0);
diff --git a/pypy/module/_multibytecodec/test/test_c_codecs.py b/pypy/module/_multibytecodec/test/test_c_codecs.py
--- a/pypy/module/_multibytecodec/test/test_c_codecs.py
+++ b/pypy/module/_multibytecodec/test/test_c_codecs.py
@@ -109,6 +109,9 @@
     assert s == 'foobar' and type(s) is str
     s = encode(c, u'\u5f95\u6cef'.encode('utf8'), 2)
     assert s == '~{abc}~}'
+    # bpo-30003
+    s = encode(c, 'ab~cd', 5)
+    assert s == 'ab~~cd'
 
 def test_encode_hz_error():
     # error

From pypy.commits at gmail.com  Wed Oct 16 13:01:28 2019
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 16 Oct 2019 10:01:28 -0700 (PDT)
Subject: [pypy-commit] pypy py3.7: hg merge py3.6
Message-ID: <5da74ce8.1c69fb81.97ba.1b9d@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.7
Changeset: r97799:5b468ca97202
Date: 2019-10-16 18:00 +0100
http://bitbucket.org/pypy/pypy/changeset/5b468ca97202/

Log:	hg merge py3.6

diff too long, truncating to 2000 out of 2237 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -50,4 +50,10 @@
 de061d87e39c7df4e436974096d7982c676a859d release-pypy3.6-v7.1.0
 784b254d669919c872a505b807db8462b6140973 release-pypy3.6-v7.1.1
 8cdda8b8cdb8ff29d9e620cccd6c5edd2f2a23ec release-pypy2.7-v7.1.1
-
+85dae4fd5c234b482feff834c73e089872194541 release-pypy2.7-v7.2.0rc0
+7ffb92269488f37c707ce66076f50ffd8613f8e2 release-pypy3.6-v7.2.0rc0
+4d6761df14ffd6f38450f183ac1fad32c946c21b release-pypy3.6-v7.2.0rc1
+5da45ced70e515f94686be0df47c59abd1348ebc release-pypy3.6-v7.2.0rc2
+4a68d8d3d2fc1faec2e83bcb4d28559099092574 release-pypy2.7-v7.2.0rc2
+4a68d8d3d2fc1faec2e83bcb4d28559099092574 release-pypy2.7-v7.2.0
+5da45ced70e515f94686be0df47c59abd1348ebc release-pypy3.6-v7.2.0
diff --git a/lib_pypy/_lzma.py b/lib_pypy/_lzma.py
--- a/lib_pypy/_lzma.py
+++ b/lib_pypy/_lzma.py
@@ -583,8 +583,16 @@
                 self.clear_input_buffer()
             elif lzs.avail_in == 0:
                 # completed successfully!
-                self.needs_input = True
                 lzs.next_in = ffi.NULL
+                if lzs.avail_out == 0:
+                    # (avail_in==0 && avail_out==0)
+                    # Maybe lzs's internal state still have a few bytes can
+                    # be output, try to output them next time.
+                    self.needs_input = False
+                    assert max_length >= 0   # if < 0, lzs.avail_out always > 0
+                else:
+                    # Input buffer exhausted, output buffer has space.
+                    self.needs_input = True
                 self.clear_input_buffer()
             else:
                 self.needs_input = False
@@ -599,9 +607,6 @@
         lzs.next_in = buf
         lzs.avail_in = buf_len
 
-        if buf_len == 0:
-            return b""
-
         bufsiz = self._bufsiz
         if not (max_length < 0 or max_length > io.DEFAULT_BUFFER_SIZE):
             bufsiz = max_length
@@ -616,7 +621,8 @@
 
         try:
             while True:
-                ret = catch_lzma_error(m.lzma_code, lzs, m.LZMA_RUN)
+                ret = catch_lzma_error(m.lzma_code, lzs, m.LZMA_RUN,
+                    ignore_buf_error=(lzs.avail_in == 0 and lzs.avail_out > 0))
                 data_size = int(ffi.cast('uintptr_t', lzs.next_out)) - int(ffi.cast('uintptr_t', orig_out))
                 # data_size is the amount lzma_code has already outputted
 
@@ -626,14 +632,17 @@
                 if ret == m.LZMA_STREAM_END:
                     self.eof = True
                     break
-                elif lzs.avail_in == 0:
-                    # it ate everything
-                    break
                 elif lzs.avail_out == 0:
+                    # Need to check lzs->avail_out before lzs->avail_in.
+                    # Maybe lzs's internal state still have a few bytes
+                    # can be output, grow the output buffer and continue
+                    # if max_lengh < 0.
                     if data_size == max_length:
                         break
                     # ran out of space in the output buffer, let's grow it
                     bufsiz += (bufsiz >> 3) + 6
+                    if max_length > 0 and bufsiz > max_length:
+                        bufsiz = max_length
                     next_out = m.realloc(orig_out, bufsiz)
                     if next_out == ffi.NULL:
                         # realloc unsuccessful
@@ -645,6 +654,9 @@
 
                     lzs.next_out = orig_out + data_size
                     lzs.avail_out = bufsiz - data_size
+                elif lzs.avail_in == 0:
+                    # it ate everything
+                    break
 
             result = ffi.buffer(orig_out, data_size)[:]
         finally:
diff --git a/pypy/doc/commandline_ref.rst b/pypy/doc/commandline_ref.rst
--- a/pypy/doc/commandline_ref.rst
+++ b/pypy/doc/commandline_ref.rst
@@ -9,3 +9,4 @@
 
    man/pypy.1.rst
    man/pypy3.1.rst
+   jit_help.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -42,7 +42,8 @@
 -------------------------------
 
 .. toctree::
-   whatsnew-pypy3-head.rst
+    whatsnew-pypy3-head.rst
+    whatsnew-pypy3-7.1.0.rst
 
 CPython 3.5 compatible versions
 -------------------------------
@@ -50,6 +51,8 @@
 .. toctree::
 
    whatsnew-pypy3-7.0.0.rst
+   whatsnew-pypy3-6.0.0.rst
+   whatsnew-pypy3-5.10.0.rst
    whatsnew-pypy3-5.9.0.rst
    whatsnew-pypy3-5.8.0.rst
    whatsnew-pypy3-5.7.0.rst
@@ -62,10 +65,4 @@
    whatsnew-pypy3-5.5.0.rst
    whatsnew-pypy3-5.1.1-alpha1.rst
 
-CPython 3.2 compatible versions
--------------------------------
 
-.. toctree::
-
-   whatsnew-pypy3-2.4.0.rst
-   whatsnew-pypy3-2.3.1.rst
diff --git a/pypy/doc/jit_help.rst b/pypy/doc/jit_help.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/jit_help.rst
@@ -0,0 +1,78 @@
+========
+JIT help
+========
+
+.. note this is from ``pypy --jit help``
+
+Advanced JIT options
+====================
+
+``<pypy> --jit`` [*options*] where *options* is a comma-separated list of
+``OPTION=VALUE``:
+
+ decay=N
+    amount to regularly decay counters by (0=none, 1000=max) (default 40)
+
+ disable_unrolling=N
+    after how many operations we should not unroll (default 200)
+
+ enable_opts=N
+    INTERNAL USE ONLY (MAY NOT WORK OR LEAD TO CRASHES): optimizations to
+    enable, or all =
+    intbounds:rewrite:virtualize:string:pure:earlyforce:heap:unroll (default
+    all)
+
+ function_threshold=N
+    number of times a function must run for it to become traced from start
+    (default 1619)
+
+ inlining=N
+    inline python functions or not (1/0) (default 1)
+
+ loop_longevity=N
+    a parameter controlling how long loops will be kept before being freed,
+    an estimate (default 1000)
+
+ max_retrace_guards=N
+    number of extra guards a retrace can cause (default 15)
+
+ max_unroll_loops=N
+    number of extra unrollings a loop can cause (default 0)
+
+ max_unroll_recursion=N
+    how many levels deep to unroll a recursive function (default 7)
+
+ retrace_limit=N
+    how many times we can try retracing before giving up (default 0)
+
+ threshold=N
+    number of times a loop has to run for it to become hot (default 1039)
+
+ trace_eagerness=N
+    number of times a guard has to fail before we start compiling a bridge
+    (default 200)
+
+ trace_limit=N
+    number of recorded operations before we abort tracing with ABORT_TOO_LONG
+    (default 6000)
+
+ vec=N
+    turn on the vectorization optimization (vecopt). Supports x86 (SSE 4.1),
+    powerpc (SVX), s390x SIMD (default 0)
+
+ vec_all=N
+    try to vectorize trace loops that occur outside of the numpypy library
+    (default 0)
+
+ vec_cost=N
+    threshold for which traces to bail. Unpacking increases the counter,
+    vector operation decrease the cost (default 0)
+
+ off
+    turn off the JIT
+ help
+    print this page
+
+The :ref:`pypyjit<jit-hooks>` module can be used to control the JIT from inside
+pypy
+
diff --git a/pypy/doc/release-v7.2.0.rst b/pypy/doc/release-v7.2.0.rst
--- a/pypy/doc/release-v7.2.0.rst
+++ b/pypy/doc/release-v7.2.0.rst
@@ -216,6 +216,7 @@
 * Add more constants to `sysconfig``. Set ``MACOSX_DEPLOYMENT_TARGET`` for
   darwin (`issue 2994`_)
 * fix ``CBuffer.buffer_attach``
+* Add ``_PyDict_GetItemWithError`` (``PyDict_GetItemWithError`` on Python3)
 
 Python 3.6 only
 ---------------
@@ -307,6 +308,7 @@
 .. _33786 : https://bugs.python.org/issue33786
 .. _32270 : https://bugs.python.org/issue32270
 .. _28691 : https://bugs.python.org/issue28691
+.. _33729 : https://bugs.python.org/issue33729
 
 .. _opencv2: https://github.com/skvark/opencv-python/
 .. _`issue 2617`: https://bitbucket.com/pypy/pypy/issues/2617
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -3,11 +3,9 @@
 ==========================
 
 .. this is a revision shortly after release-pypy-7.2.0
-.. startrev: 78cd4acbcbec 
+.. startrev: a511d86377d6 
 
+.. branch: fix-descrmismatch-crash
 
-.. branch: json-decoder-maps
+Fix segfault when calling descr-methods with no arguments
 
-Much faster and more memory-efficient JSON decoding. The resulting
-dictionaries that come out of the JSON decoder have faster lookups too.
-
diff --git a/pypy/doc/whatsnew-pypy2-7.2.0.rst b/pypy/doc/whatsnew-pypy2-7.2.0.rst
--- a/pypy/doc/whatsnew-pypy2-7.2.0.rst
+++ b/pypy/doc/whatsnew-pypy2-7.2.0.rst
@@ -17,10 +17,6 @@
 
 Add ``DateTime_FromTimestamp`` and ``Date_FromTimestamp``
 
-.. branch: issue2968
-
-Fix segfault in cpyext_tp_new_tupl
-
 .. branch: semlock-deadlock
 
 Test and reduce the probability of a deadlock when acquiring a semaphore by
@@ -79,6 +75,8 @@
 
 Update _ssl on macos to statically link to openssl-1.1.1c
 
-.. branch: more-cpyext
+.. branch: json-decoder-maps
 
-Add more datetime C functions and definitions
+Much faster and more memory-efficient JSON decoding. The resulting
+dictionaries that come out of the JSON decoder have faster lookups too.
+
diff --git a/pypy/doc/whatsnew-pypy3-2.3.1.rst b/pypy/doc/whatsnew-pypy3-2.3.1.rst
deleted file mode 100644
--- a/pypy/doc/whatsnew-pypy3-2.3.1.rst
+++ /dev/null
@@ -1,6 +0,0 @@
-=========================
-What's new in PyPy3 2.3.1
-=========================
-
-.. this is a revision shortly after pypy3-release-2.3.x
-.. startrev: 0137d8e6657d
diff --git a/pypy/doc/whatsnew-pypy3-2.4.0.rst b/pypy/doc/whatsnew-pypy3-2.4.0.rst
deleted file mode 100644
--- a/pypy/doc/whatsnew-pypy3-2.4.0.rst
+++ /dev/null
@@ -1,6 +0,0 @@
-=========================
-What's new in PyPy3 2.4.0
-=========================
-
-.. this is a revision shortly after pypy3-release-2.4.x
-.. startrev: 12b940544622
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -394,6 +394,8 @@
 
 @specialize.memo()
 def wrappable_class_name(Class):
+    if 'exact_class_applevel_name' in Class.__dict__:
+        return Class.exact_class_applevel_name
     try:
         return Class.typedef.name
     except AttributeError:
@@ -1232,8 +1234,11 @@
         from pypy.interpreter.generator import GeneratorIterator
         return isinstance(w_obj, GeneratorIterator)
 
+    def callable_w(self, w_obj):
+        return self.lookup(w_obj, "__call__") is not None
+
     def callable(self, w_obj):
-        return self.newbool(self.lookup(w_obj, "__call__") is not None)
+        return self.newbool(self.callable_w(w_obj))
 
     def issequence_w(self, w_obj):
         flag = self.type(w_obj).flag_map_or_seq
@@ -1787,8 +1792,7 @@
     def convert_arg_to_w_unicode(self, w_obj, strict=None):
         # XXX why convert_to_w_unicode does something slightly different?
         from pypy.objspace.std.unicodeobject import W_UnicodeObject
-        # for z_translation tests
-        if hasattr(self, 'is_fake_objspace'): return self.newtext("foobar")
+        assert not hasattr(self, 'is_fake_objspace')
         return W_UnicodeObject.convert_arg_to_w_unicode(self, w_obj, strict)
 
     def utf8_len_w(self, w_obj):
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -23,8 +23,9 @@
     # XXX [fijal] but they're not. is_being_profiled is guarded a bit all
     #     over the place as well as w_tracefunc
 
-    _immutable_fields_ = ['profilefunc?', 'w_tracefunc?',
-                          'w_coroutine_wrapper_fn?']
+    _immutable_fields_ = [
+        'profilefunc?', 'w_tracefunc?', 'w_coroutine_wrapper_fn?',
+        'w_asyncgen_firstiter_fn?', 'w_asyncgen_finalizer_fn?']
 
     def __init__(self, space):
         self.space = space
@@ -41,6 +42,8 @@
         self.thread_disappeared = False   # might be set to True after os.fork()
         self.w_coroutine_wrapper_fn = None
         self.in_coroutine_wrapper = False
+        self.w_asyncgen_firstiter_fn = None
+        self.w_asyncgen_finalizer_fn = None
 
     @staticmethod
     def _mark_thread_disappeared(space):
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -755,6 +755,7 @@
                     self.func__args__ = func
                 elif unwrap_spec == [self_type, ObjSpace, Arguments]:
                     self.__class__ = BuiltinCodePassThroughArguments1
+                    self.descr_reqcls = self_type
                     miniglobals = {'func': func, 'self_type': self_type}
                     d = {}
                     source = """if 1:
@@ -800,10 +801,7 @@
         except DescrMismatch:
             if w_obj is not None:
                 args = args.prepend(w_obj)
-            return scope_w[0].descr_call_mismatch(space,
-                                                  self.descrmismatch_op,
-                                                  self.descr_reqcls,
-                                                  args)
+            return self._type_unwrap_mismatch(space, args)
         except Exception as e:
             self.handle_exception(space, e)
             w_result = None
@@ -811,6 +809,15 @@
             w_result = space.w_None
         return w_result
 
+    def _type_unwrap_mismatch(self, space, args):
+        w_obj = args.firstarg()
+        if w_obj is None:
+            raise oefmt(space.w_SystemError, "unexpected DescrMismatch error")
+        return w_obj.descr_call_mismatch(space,
+                                         self.descrmismatch_op,
+                                         self.descr_reqcls,
+                                         args)
+
     def handle_exception(self, space, e):
         try:
             if not we_are_translated():
@@ -833,10 +840,7 @@
         try:
             w_result = self.func__args__(space, args)
         except DescrMismatch:
-            return args.firstarg().descr_call_mismatch(space,
-                                                  self.descrmismatch_op,
-                                                  self.descr_reqcls,
-                                                  args)
+            return self._type_unwrap_mismatch(space, args)
         except Exception as e:
             self.handle_exception(space, e)
             w_result = None
@@ -854,10 +858,7 @@
         try:
             w_result = self.func__args__(space, w_obj, args)
         except DescrMismatch:
-            return args.firstarg().descr_call_mismatch(space,
-                                                  self.descrmismatch_op,
-                                                  self.descr_reqcls,
-                                                  args.prepend(w_obj))
+            return self._type_unwrap_mismatch(space, args.prepend(w_obj))
         except Exception as e:
             self.handle_exception(space, e)
             w_result = None
@@ -897,9 +898,7 @@
         try:
             w_result = self.fastfunc_1(space, w1)
         except DescrMismatch:
-            return w1.descr_call_mismatch(space,
-                                          self.descrmismatch_op,
-                                          self.descr_reqcls,
+            return self._type_unwrap_mismatch(space,
                                           Arguments(space, [w1]))
         except Exception as e:
             self.handle_exception(space, e)
@@ -923,9 +922,7 @@
         try:
             w_result = self.fastfunc_2(space, w1, w2)
         except DescrMismatch:
-            return w1.descr_call_mismatch(space,
-                                          self.descrmismatch_op,
-                                          self.descr_reqcls,
+            return self._type_unwrap_mismatch(space,
                                           Arguments(space, [w1, w2]))
         except Exception as e:
             self.handle_exception(space, e)
@@ -950,9 +947,7 @@
         try:
             w_result = self.fastfunc_3(space, w1, w2, w3)
         except DescrMismatch:
-            return w1.descr_call_mismatch(space,
-                                          self.descrmismatch_op,
-                                          self.descr_reqcls,
+            return self._type_unwrap_mismatch(space,
                                           Arguments(space, [w1, w2, w3]))
         except Exception as e:
             self.handle_exception(space, e)
@@ -978,9 +973,7 @@
         try:
             w_result = self.fastfunc_4(space, w1, w2, w3, w4)
         except DescrMismatch:
-            return w1.descr_call_mismatch(space,
-                                          self.descrmismatch_op,
-                                          self.descr_reqcls,
+            return self._type_unwrap_mismatch(space,
                                           Arguments(space,
                                                     [w1, w2, w3, w4]))
         except Exception as e:
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -11,8 +11,6 @@
 class GeneratorOrCoroutine(W_Root):
     _immutable_fields_ = ['pycode']
 
-    w_yielded_from = None
-
     def __init__(self, frame, name=None, qualname=None):
         self.space = frame.space
         self.frame = frame     # turned into None when frame_finished_execution
@@ -84,7 +82,7 @@
                 operr = OperationError(space.w_StopIteration, space.w_None)
             raise operr
 
-        w_result = self._invoke_execute_frame(frame, w_arg_or_err)
+        w_result = self._invoke_execute_frame(w_arg_or_err)
         assert w_result is not None
 
         # if the frame is now marked as finished, it was RETURNed from
@@ -102,8 +100,9 @@
         else:
             return w_result     # YIELDed
 
-    def _invoke_execute_frame(self, frame, w_arg_or_err):
+    def _invoke_execute_frame(self, w_arg_or_err):
         space = self.space
+        frame = self.frame
         if self.running:
             raise oefmt(space.w_ValueError, "%s already executing", self.KIND)
         ec = space.getexecutioncontext()
@@ -126,7 +125,7 @@
                             self.KIND)
         self.running = True
         try:
-            w_result = frame.execute_frame(self, w_arg_or_err)
+            w_result = frame.execute_frame(w_arg_or_err)
         except OperationError as e:
             # errors finish a frame
             try:
@@ -149,68 +148,19 @@
             ec.set_sys_exc_info(current_exc_info)
         return w_result
 
-    def resume_execute_frame(self, frame, w_arg_or_err):
-        # Called from execute_frame() just before resuming the bytecode
-        # interpretation.
-        space = self.space
-        w_yf = self.w_yielded_from
-        if w_yf is not None:
-            self.w_yielded_from = None
-            try:
-                self.next_yield_from(frame, w_yf, w_arg_or_err)
-            except OperationError as operr:
-                operr.record_context(space, space.getexecutioncontext())
-                return frame.handle_generator_error(operr)
-            # Normal case: the call above raises Yield.
-            # We reach this point if the iterable is exhausted.
-            last_instr = jit.promote(frame.last_instr)
-            assert last_instr & 1 == 0
-            assert last_instr >= 0
-            return r_uint(last_instr + 2)
+    def get_delegate(self):
+        if self.frame is None:
+            return None
+        return self.frame.w_yielding_from
 
-        if isinstance(w_arg_or_err, SApplicationException):
-            return frame.handle_generator_error(w_arg_or_err.operr)
+    def descr_delegate(self, space):
+        w_yf = self.get_delegate()
+        if w_yf is None:
+            return space.w_None
+        return w_yf
 
-        last_instr = jit.promote(frame.last_instr)
-        if last_instr != -1:
-            assert last_instr & 1 == 0
-            frame.pushvalue(w_arg_or_err)
-            return r_uint(last_instr + 2)
-        else:
-            return r_uint(0)
-
-    def next_yield_from(self, frame, w_yf, w_inputvalue_or_err):
-        """Fetch the next item of the current 'yield from', push it on
-        the frame stack, and raises Yield.  If there isn't one, push
-        w_stopiteration_value and returns.  May also just raise.
-        """
-        space = self.space
-        try:
-            if isinstance(w_yf, GeneratorOrCoroutine):
-                w_retval = w_yf.send_ex(w_inputvalue_or_err)
-            elif isinstance(w_yf, AsyncGenASend):   # performance only
-                w_retval = w_yf.do_send(w_inputvalue_or_err)
-            elif space.is_w(w_inputvalue_or_err, space.w_None):
-                w_retval = space.next(w_yf)
-            else:
-                w_retval = delegate_to_nongen(space, w_yf, w_inputvalue_or_err)
-        except OperationError as e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            frame._report_stopiteration_sometimes(w_yf, e)
-            try:
-                w_stop_value = space.getattr(e.get_w_value(space),
-                                             space.newtext("value"))
-            except OperationError as e:
-                if not e.match(space, space.w_AttributeError):
-                    raise
-                w_stop_value = space.w_None
-            frame.pushvalue(w_stop_value)
-            return
-        else:
-            frame.pushvalue(w_retval)
-            self.w_yielded_from = w_yf
-            raise Yield
+    def set_delegate(self, w_delegate):
+        self.frame.w_yielding_from = w_delegate
 
     def _leak_stopiteration(self, e):
         # Check for __future__ generator_stop and conditionally turn
@@ -259,8 +209,8 @@
         operr = OperationError(w_type, w_val, tb)
         operr.normalize_exception(space)
 
-        # note: w_yielded_from is always None if 'self.running'
-        if (self.w_yielded_from is not None and
+        # note: _w_yielded_from is always None if 'self.running'
+        if (self.get_delegate() is not None and
                     operr.match(space, space.w_GeneratorExit)):
             try:
                 self._gen_close_iter(space)
@@ -276,8 +226,8 @@
 
     def _gen_close_iter(self, space):
         assert not self.running
-        w_yf = self.w_yielded_from
-        self.w_yielded_from = None
+        w_yf = self.get_delegate()
+        self.set_delegate(None)
         self.running = True
         try:
             gen_close_iter(space, w_yf)
@@ -290,8 +240,8 @@
             return     # nothing to do in this case
         space = self.space
         operr = None
-        # note: w_yielded_from is always None if 'self.running'
-        w_yf = self.w_yielded_from
+        # note: _w_yielded_from is always None if 'self.running'
+        w_yf = self.get_delegate()
         if w_yf is not None:
             try:
                 self._gen_close_iter(space)
@@ -375,7 +325,7 @@
     # generate 2 versions of the function and 2 jit drivers.
     def _create_unpack_into():
         jitdriver = jit.JitDriver(greens=['pycode'],
-                                  reds=['self', 'frame', 'results'],
+                                  reds='auto',
                                   name='unpack_into')
 
         def unpack_into(self, results):
@@ -386,12 +336,10 @@
                 return
             pycode = self.pycode
             while True:
-                jitdriver.jit_merge_point(self=self, frame=frame,
-                                          results=results, pycode=pycode)
+                jitdriver.jit_merge_point(pycode=pycode)
                 space = self.space
                 try:
-                    w_result = self._invoke_execute_frame(
-                                            frame, space.w_None)
+                    w_result = self._invoke_execute_frame(space.w_None)
                 except OperationError as e:
                     if not e.match(space, space.w_StopIteration):
                         raise
@@ -489,27 +437,6 @@
         else:
             space.call_function(w_close)
 
-def delegate_to_nongen(space, w_yf, w_inputvalue_or_err):
-    # invoke a "send" or "throw" by method name to a non-generator w_yf
-    if isinstance(w_inputvalue_or_err, SApplicationException):
-        operr = w_inputvalue_or_err.operr
-        try:
-            w_meth = space.getattr(w_yf, space.newtext("throw"))
-        except OperationError as e:
-            if not e.match(space, space.w_AttributeError):
-                raise
-            raise operr
-        # bah, CPython calls here with the exact same arguments as
-        # originally passed to throw().  In our case it is far removed.
-        # Let's hope nobody will complain...
-        operr.normalize_exception(space)
-        w_exc = operr.w_type
-        w_val = operr.get_w_value(space)
-        w_tb  = operr.get_w_traceback(space)
-        return space.call_function(w_meth, w_exc, w_val, w_tb)
-    else:
-        return space.call_method(w_yf, "send", w_inputvalue_or_err)
-
 def gen_is_coroutine(w_obj):
     return (isinstance(w_obj, GeneratorIterator) and
             (w_obj.pycode.co_flags & consts.CO_ITERABLE_COROUTINE) != 0)
@@ -594,15 +521,15 @@
         if self.hooks_inited:
             return
         self.hooks_inited = True
-
-        self.w_finalizer = self.space.appexec([], '''():
-            import sys
-            hooks = sys.get_asyncgen_hooks()
-            return hooks.finalizer''')
+        ec = self.space.getexecutioncontext()
+        self.w_finalizer = ec.w_asyncgen_finalizer_fn
+        w_firstiter = ec.w_asyncgen_firstiter_fn
+        if w_firstiter is not None:
+            self.space.call_function(w_firstiter, self)
 
     def _finalize_(self):
         if self.frame is not None and self.frame.lastblock is not None:
-            if self.w_finalizer is not self.space.w_None:
+            if self.w_finalizer is not None:
                 # XXX: this is a hack to resurrect the weakref that was cleared
                 # before running _finalize_()
                 if self.space.config.translation.rweakref:
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -68,6 +68,7 @@
     frame_finished_execution = False
     f_generator_wref         = rweakref.dead_ref  # for generators/coroutines
     f_generator_nowref       = None               # (only one of the two attrs)
+    w_yielding_from = None
     last_instr               = -1
     f_backref                = jit.vref_None
 
@@ -319,9 +320,40 @@
                 ec.in_coroutine_wrapper = False
         return w_gen
 
-    def execute_frame(self, in_generator=None, w_arg_or_err=None):
+    def resume_execute_frame(self, w_arg_or_err):
+        # Called from execute_frame() just before resuming the bytecode
+        # interpretation.
+        from pypy.interpreter.pyopcode import SApplicationException
+        space = self.space
+        w_yf = self.w_yielding_from
+        if w_yf is not None:
+            self.w_yielding_from = None
+            try:
+                self.next_yield_from(w_yf, w_arg_or_err)
+            except OperationError as operr:
+                operr.record_context(space, space.getexecutioncontext())
+                return self.handle_generator_error(operr)
+            # Normal case: the call above raises Yield.
+            # We reach this point if the iterable is exhausted.
+            last_instr = jit.promote(self.last_instr)
+            assert last_instr & 1 == 0
+            assert last_instr >= 0
+            return r_uint(last_instr + 2)
+
+        if isinstance(w_arg_or_err, SApplicationException):
+            return self.handle_generator_error(w_arg_or_err.operr)
+
+        last_instr = jit.promote(self.last_instr)
+        if last_instr != -1:
+            assert last_instr & 1 == 0
+            self.pushvalue(w_arg_or_err)
+            return r_uint(last_instr + 2)
+        else:
+            return r_uint(0)
+
+    def execute_frame(self, w_arg_or_err=None):
         """Execute this frame.  Main entry point to the interpreter.
-        'in_generator' is non-None iff we are starting or resuming
+        'w_arg_or_err' is non-None iff we are starting or resuming
         a generator or coroutine frame; in that case, w_arg_or_err
         is the input argument -or- an SApplicationException instance.
         """
@@ -343,12 +375,11 @@
             # the YIELD_VALUE/YIELD_FROM instruction.
             try:
                 try:
-                    if in_generator is None:
+                    if w_arg_or_err is None:
                         assert self.last_instr == -1
                         next_instr = r_uint(0)
                     else:
-                        next_instr = in_generator.resume_execute_frame(
-                                                        self, w_arg_or_err)
+                        next_instr = self.resume_execute_frame(w_arg_or_err)
                 except pyopcode.Yield:
                     w_exitvalue = self.popvalue()
                 else:
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1113,24 +1113,51 @@
             self.pushvalue(w_value)
         raise Yield
 
+    def next_yield_from(self, w_yf, w_inputvalue_or_err):
+        """Fetch the next item of the current 'yield from', push it on
+        the frame stack, and raises Yield.  If there isn't one, push
+        w_stopiteration_value and returns.  May also just raise.
+        """
+        from pypy.interpreter.generator import (
+            GeneratorOrCoroutine, AsyncGenASend)
+        space = self.space
+        try:
+            if isinstance(w_yf, GeneratorOrCoroutine):
+                w_retval = w_yf.send_ex(w_inputvalue_or_err)
+            elif isinstance(w_yf, AsyncGenASend):   # performance only
+                w_retval = w_yf.do_send(w_inputvalue_or_err)
+            elif space.is_w(w_inputvalue_or_err, space.w_None):
+                w_retval = space.next(w_yf)
+            else:
+                w_retval = delegate_to_nongen(space, w_yf, w_inputvalue_or_err)
+        except OperationError as e:
+            if not e.match(space, space.w_StopIteration):
+                raise
+            self._report_stopiteration_sometimes(w_yf, e)
+            try:
+                w_stop_value = space.getattr(e.get_w_value(space),
+                                             space.newtext("value"))
+            except OperationError as e:
+                if not e.match(space, space.w_AttributeError):
+                    raise
+                w_stop_value = space.w_None
+            self.pushvalue(w_stop_value)
+            return
+        else:
+            self.pushvalue(w_retval)
+            self.w_yielding_from = w_yf
+            raise Yield
+
     def YIELD_FROM(self, oparg, next_instr):
         # Unlike CPython, we handle this not by repeating the same
         # bytecode over and over until the inner iterator is exhausted.
-        # Instead, we directly set the generator's w_yielded_from.
-        # This asks generator.resume_execute_frame() to exhaust that
+        # Instead, we set w_yielding_from.
+        # This asks resume_execute_frame() to exhaust that
         # sub-iterable first before continuing on the next bytecode.
-        in_generator = self.get_generator()
-        if in_generator is None:
-            # Issue #2768: rare case involving __del__ methods.
-            # XXX This is a workaround, not a proper solution.
-            raise oefmt(self.space.w_RuntimeError,
-                        "PyPy limitation: cannot use 'yield from' or 'await' "
-                        "in a generator/coroutine that has been partially "
-                        "deallocated already, typically seen via __del__")
         w_inputvalue = self.popvalue()    # that's always w_None, actually
         w_gen = self.popvalue()
         #
-        in_generator.next_yield_from(self, w_gen, w_inputvalue)
+        self.next_yield_from(w_gen, w_inputvalue)
         # Common case: the call above raises Yield.
         # If instead the iterable is empty, next_yield_from() pushed the
         # final result and returns.  In that case, we can just continue
@@ -1556,7 +1583,7 @@
         w_iterable = self.popvalue()
         w_iter = get_awaitable_iter(self.space, w_iterable)
         if isinstance(w_iter, Coroutine):
-            if w_iter.w_yielded_from is not None:
+            if w_iter.get_delegate() is not None:
                 # 'w_iter' is a coroutine object that is being awaited,
                 # '.w_yielded_from' is the current awaitable being awaited on.
                 raise oefmt(self.space.w_RuntimeError,
@@ -1694,6 +1721,27 @@
         else:
             self.MISSING_OPCODE(oparg, next_instr)
 
+def delegate_to_nongen(space, w_yf, w_inputvalue_or_err):
+    # invoke a "send" or "throw" by method name to a non-generator w_yf
+    if isinstance(w_inputvalue_or_err, SApplicationException):
+        operr = w_inputvalue_or_err.operr
+        try:
+            w_meth = space.getattr(w_yf, space.newtext("throw"))
+        except OperationError as e:
+            if not e.match(space, space.w_AttributeError):
+                raise
+            raise operr
+        # bah, CPython calls here with the exact same arguments as
+        # originally passed to throw().  In our case it is far removed.
+        # Let's hope nobody will complain...
+        operr.normalize_exception(space)
+        w_exc = operr.w_type
+        w_val = operr.get_w_value(space)
+        w_tb = operr.get_w_traceback(space)
+        return space.call_function(w_meth, w_exc, w_val, w_tb)
+    else:
+        return space.call_method(w_yf, "send", w_inputvalue_or_err)
+
 
 ### ____________________________________________________________ ###
 
diff --git a/pypy/interpreter/test/apptest_coroutine.py b/pypy/interpreter/test/apptest_coroutine.py
--- a/pypy/interpreter/test/apptest_coroutine.py
+++ b/pypy/interpreter/test/apptest_coroutine.py
@@ -633,16 +633,12 @@
         a2 = g.aclose()
     sys.set_asyncgen_hooks(finalizer=_finalize)
     assert state == 0
-    try:
+    with pytest.raises(StopIteration):
         a.send(None)
-    except StopIteration:
-        pass
     assert a2.send(None) == 'coro'
     assert state == 1
-    try:
+    with pytest.raises(StopIteration):
         a2.send(None)
-    except StopIteration:
-        pass
     assert state == 2
     sys.set_asyncgen_hooks(None, None)
 
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -1095,6 +1095,29 @@
         # white-box check for opt
         assert called[0] is args
 
+    def test_base_regular_descr_mismatch(self):
+        space = self.space
+
+        def f():
+            raise gateway.DescrMismatch
+
+        w_f = space.wrap(gateway.interp2app_temp(f,
+                         unwrap_spec=[]))
+        args = argument.Arguments(space, [])
+        space.raises_w(space.w_SystemError, space.call_args, w_f, args)
+
+    def test_pass_trough_arguments0_descr_mismatch(self):
+        space = self.space
+
+        def f(space, __args__):
+            raise gateway.DescrMismatch
+
+        w_f = space.wrap(gateway.interp2app_temp(f,
+                         unwrap_spec=[gateway.ObjSpace,
+                                      gateway.Arguments]))
+        args = argument.Arguments(space, [])
+        space.raises_w(space.w_SystemError, space.call_args, w_f, args)
+
 
 class AppTestKeywordsToBuiltinSanity(object):
     def test_type(self):
@@ -1134,3 +1157,26 @@
 
         d.update(**{clash: 33})
         dict.update(d, **{clash: 33})
+
+
+
+class AppTestFastPathCrash(object):
+    def setup_class(cls):
+        cls.w_runappdirect = cls.space.wrap(cls.runappdirect)
+
+    def test_fast_path_crash(self):
+        # issue bb-3091 crash in BuiltinCodePassThroughArguments0.funcrun
+        for obj in (dict, set):
+            with raises(TypeError) as excinfo:
+                if self.runappdirect:
+                    import platform
+                    if platform.python_implementation() == 'PyPy':
+                        msg_fmt = "%s instance as first argument (got %s"
+                    else:
+                        msg_fmt = "'%s' object but received a '%s'"
+                    obj.__init__(0)
+                else:
+                    msg_fmt = "'%s' object expected, got '%s'"
+                    obj.__init__.im_func(0)
+            msg = msg_fmt %(obj.__name__, 'int')
+            assert msg in str(excinfo.value)
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -838,7 +838,7 @@
     gi_running = interp_attrproperty('running', cls=GeneratorIterator, wrapfn="newbool"),
     gi_frame   = GetSetProperty(GeneratorIterator.descr_gicr_frame),
     gi_code    = interp_attrproperty_w('pycode', cls=GeneratorIterator),
-    gi_yieldfrom=interp_attrproperty_w('w_yielded_from', cls=GeneratorIterator),
+    gi_yieldfrom=GetSetProperty(GeneratorIterator.descr_delegate),
     __name__   = GetSetProperty(GeneratorIterator.descr__name__,
                                 GeneratorIterator.descr_set__name__),
     __qualname__ = GetSetProperty(GeneratorIterator.descr__qualname__,
@@ -862,7 +862,7 @@
     cr_running = interp_attrproperty('running', cls=Coroutine, wrapfn="newbool"),
     cr_frame   = GetSetProperty(Coroutine.descr_gicr_frame),
     cr_code    = interp_attrproperty_w('pycode', cls=Coroutine),
-    cr_await   = interp_attrproperty_w('w_yielded_from', cls=Coroutine),
+    cr_await=GetSetProperty(Coroutine.descr_delegate),
     __name__   = GetSetProperty(Coroutine.descr__name__,
                                 Coroutine.descr_set__name__,
                                 doc="name of the coroutine"),
@@ -890,7 +890,7 @@
     ag_running = interp_attrproperty('running', cls=AsyncGenerator, wrapfn="newbool"),
     ag_frame   = GetSetProperty(AsyncGenerator.descr_gicr_frame),
     ag_code    = interp_attrproperty_w('pycode', cls=AsyncGenerator),
-    ag_await   = interp_attrproperty_w('w_yielded_from', cls=AsyncGenerator),
+    ag_await=GetSetProperty(AsyncGenerator.descr_delegate),
     __name__   = GetSetProperty(AsyncGenerator.descr__name__,
                                 AsyncGenerator.descr_set__name__,
                                 doc="name of the async generator"),
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -375,6 +375,14 @@
         return res_utf8, len(res), size
 
 def str_decode_utf8(s, errors, final, errorhandler, allow_surrogates=False):
+    try:
+        # fast version first
+        return s, rutf8.check_utf8(s, allow_surrogates=allow_surrogates), len(s)
+    except rutf8.CheckError:
+        return _str_decode_utf8_slowpath(
+            s, errors, final, errorhandler, allow_surrogates=allow_surrogates)
+
+def _str_decode_utf8_slowpath(s, errors, final, errorhandler, allow_surrogates):
     """ Same as checking for the valid utf8, but we know the utf8 is not
     valid so we're trying to either raise or pack stuff with error handler.
     The key difference is that this is call_may_force
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -737,17 +737,10 @@
         errors = 'strict'
     final = space.is_true(w_final)
     state = space.fromcache(CodecState)
-    # call the fast version for checking
-    try:
-        lgt = rutf8.check_utf8(string, allow_surrogates=False)
-    except rutf8.CheckError:
-        res, lgt, pos = unicodehelper.str_decode_utf8(string,
-            errors, final, state.decode_error_handler)
-        return space.newtuple([space.newutf8(res, lgt),
-                               space.newint(pos)])
-    else:
-        return space.newtuple([space.newutf8(string, lgt),
-                               space.newint(len(string))])
+    res, lgt, pos = unicodehelper.str_decode_utf8(string,
+        errors, final, state.decode_error_handler)
+    return space.newtuple([space.newutf8(res, lgt),
+                           space.newint(pos)])
 
 @unwrap_spec(data='bufferstr', errors='text_or_none', byteorder=int,
              w_final=WrappedDefault(False))
diff --git a/pypy/module/_csv/interp_csv.py b/pypy/module/_csv/interp_csv.py
--- a/pypy/module/_csv/interp_csv.py
+++ b/pypy/module/_csv/interp_csv.py
@@ -1,3 +1,4 @@
+from rpython.rlib import rutf8
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.typedef import TypeDef, interp_attrproperty
@@ -47,24 +48,26 @@
     if w_src is None:
         return default
     try:
-        return space.realunicode_w(w_src)
+        return space.text_w(w_src)
     except OperationError as e:
         if e.match(space, space.w_TypeError):
             raise oefmt(space.w_TypeError, '"%s" must be a string', attrname)
         raise
 
-def _get_char(space, w_src, default, name):
+def _get_codepoint(space, w_src, default, name):
     if w_src is None:
         return default
     if space.is_w(w_src, space.w_None):
-        return u'\0'
+        return 0
     if not space.isinstance_w(w_src, space.w_unicode):
         raise oefmt(space.w_TypeError, '"%s" must be string, not %T', name, w_src)
-    src = space.realunicode_w(w_src)
-    if len(src) == 1:
-        return src[0]
+    src, length = space.utf8_len_w(w_src)
+    if length == 1:
+        res = rutf8.codepoint_at_pos(src, 0)
+        assert res >= 0
+        return res
     if len(src) == 0:
-        return u'\0'
+        return 0
     raise oefmt(space.w_TypeError, '"%s" must be a 1-character string', name)
 
 def _build_dialect(space, w_dialect, w_delimiter, w_doublequote,
@@ -104,11 +107,11 @@
             w_strict = _fetch(space, w_dialect, 'strict')
 
     dialect = W_Dialect()
-    dialect.delimiter = _get_char(space, w_delimiter, u',', 'delimiter')
+    dialect.delimiter = _get_codepoint(space, w_delimiter, ord(u','), 'delimiter')
     dialect.doublequote = _get_bool(space, w_doublequote, True)
-    dialect.escapechar = _get_char(space, w_escapechar, u'\0', 'escapechar')
-    dialect.lineterminator = _get_str(space, w_lineterminator, u'\r\n', 'lineterminator')
-    dialect.quotechar = _get_char(space, w_quotechar, u'"', 'quotechar')
+    dialect.escapechar = _get_codepoint(space, w_escapechar, ord(u'\0'), 'escapechar')
+    dialect.lineterminator = _get_str(space, w_lineterminator, '\r\n', 'lineterminator')
+    dialect.quotechar = _get_codepoint(space, w_quotechar, ord(u'"'), 'quotechar')
     tmp_quoting = _get_int(space, w_quoting, QUOTE_MINIMAL, 'quoting')
     dialect.skipinitialspace = _get_bool(space, w_skipinitialspace, False)
     dialect.strict = _get_bool(space, w_strict, False)
@@ -117,13 +120,13 @@
     if not (0 <= tmp_quoting < 4):
         raise oefmt(space.w_TypeError, 'bad "quoting" value')
 
-    if dialect.delimiter == u'\0':
+    if dialect.delimiter == 0:
         raise oefmt(space.w_TypeError,
                     '"delimiter" must be a 1-character string')
 
     if space.is_w(w_quotechar, space.w_None) and w_quoting is None:
         tmp_quoting = QUOTE_NONE
-    if tmp_quoting != QUOTE_NONE and dialect.quotechar == u'\0':
+    if tmp_quoting != QUOTE_NONE and dialect.quotechar == 0:
         raise oefmt(space.w_TypeError,
                     "quotechar must be set if quoting enabled")
     dialect.quoting = tmp_quoting
@@ -158,14 +161,20 @@
 
 
 def _get_escapechar(space, dialect):
-    if dialect.escapechar == u'\0':
+    if dialect.escapechar == 0:
         return space.w_None
-    return space.newtext(dialect.escapechar)
+    s = rutf8.unichr_as_utf8(dialect.escapechar)
+    return space.newutf8(s, 1)
 
 def _get_quotechar(space, dialect):
-    if dialect.quotechar == u'\0':
+    if dialect.quotechar == 0:
         return space.w_None
-    return space.newtext(dialect.quotechar)
+    s = rutf8.unichr_as_utf8(dialect.quotechar)
+    return space.newutf8(s, 1)
+
+def _get_delimiter(space, dialect):
+    s = rutf8.unichr_as_utf8(dialect.delimiter)
+    return space.newutf8(s, 1)
 
 
 W_Dialect.typedef = TypeDef(
@@ -173,8 +182,7 @@
         __new__ = interp2app(W_Dialect___new__),
         __reduce_ex__ = interp2app(W_Dialect.reduce_ex_w),
 
-        delimiter        = interp_attrproperty('delimiter', W_Dialect,
-            wrapfn='newtext'),
+        delimiter        = GetSetProperty(_get_delimiter, cls=W_Dialect),
         doublequote      = interp_attrproperty('doublequote', W_Dialect,
             wrapfn='newbool'),
         escapechar       = GetSetProperty(_get_escapechar, cls=W_Dialect),
diff --git a/pypy/module/_csv/interp_reader.py b/pypy/module/_csv/interp_reader.py
--- a/pypy/module/_csv/interp_reader.py
+++ b/pypy/module/_csv/interp_reader.py
@@ -1,5 +1,4 @@
-from rpython.rlib.rstring import UnicodeBuilder
-from rpython.rlib.rutf8 import Utf8StringIterator
+from rpython.rlib.rutf8 import Utf8StringIterator, Utf8StringBuilder
 from rpython.rlib import objectmodel
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError
@@ -22,6 +21,7 @@
         self.dialect = dialect
         self.w_iter = w_iter
         self.line_num = 0
+        self.sizehint = 1  # just used for first line
 
     def iter_w(self):
         return self
@@ -38,22 +38,21 @@
         assert field_builder is not None
         if field_builder.getlength() >= field_limit.limit:
             raise self.error(u"field larger than field limit")
-        field_builder.append(c)
+        field_builder.append_code(c)
 
     def save_field(self, field_builder):
         space = self.space
         field = field_builder.build()
+        w_obj = space.newutf8(field, field_builder.getlength())
         if self.numeric_field:
             self.numeric_field = False
-            w_obj = space.call_function(space.w_float, space.newtext(field))
-        else:
-            w_obj = space.newtext(field)
+            w_obj = space.call_function(space.w_float, w_obj)
         self.fields_w.append(w_obj)
 
     def next_w(self):
         space = self.space
         dialect = self.dialect
-        self.fields_w = []
+        self.fields_w = objectmodel.newlist_hint(self.sizehint)
         self.numeric_field = False
         field_builder = None  # valid iff state not in [START_RECORD, EAT_CRNL]
         state = START_RECORD
@@ -79,13 +78,11 @@
                                  u"(did you open the file in text mode?")
             line = space.utf8_w(w_line)
             for c in Utf8StringIterator(line):
-                # XXX rewrite this to use c (as int) not unichr(c)
-                c = unichr(c)
-                if c == '\0':
+                if c == 0:
                     raise self.error(u"line contains NULL byte")
 
                 if state == START_RECORD:
-                    if c == b'\n' or c == b'\r':
+                    if c == ord(u'\n') or c == ord(u'\r'):
                         state = EAT_CRNL
                         continue
                     # normal character - handle as START_FIELD
@@ -93,9 +90,9 @@
                     # fall-through to the next case
 
                 if state == START_FIELD:
-                    field_builder = UnicodeBuilder(64)
+                    field_builder = Utf8StringBuilder(64)
                     # expecting field
-                    if c == u'\n' or c == u'\r':
+                    if c == ord(u'\n') or c == ord(u'\r'):
                         # save empty field
                         self.save_field(field_builder)
                         state = EAT_CRNL
@@ -106,7 +103,7 @@
                     elif c == dialect.escapechar:
                         # possible escaped character
                         state = ESCAPED_CHAR
-                    elif c == u' ' and dialect.skipinitialspace:
+                    elif c == ord(u' ') and dialect.skipinitialspace:
                         # ignore space at start of field
                         pass
                     elif c == dialect.delimiter:
@@ -120,7 +117,7 @@
                         state = IN_FIELD
 
                 elif state == ESCAPED_CHAR:
-                    if c in '\n\r':
+                    if c == ord(u'\n') or c == ord(u'\r'):
                         self.add_char(field_builder, c)
                         state = AFTER_ESCAPED_CRNL
                     else:
@@ -129,7 +126,7 @@
 
                 elif state == IN_FIELD or state == AFTER_ESCAPED_CRNL:
                     # in unquoted field
-                    if c == u'\n' or c == u'\r':
+                    if c == ord(u'\n') or c == ord(u'\r'):
                         # end of line
                         self.save_field(field_builder)
                         state = EAT_CRNL
@@ -176,7 +173,7 @@
                         # save field - wait for new field
                         self.save_field(field_builder)
                         state = START_FIELD
-                    elif c == u'\n' or c == u'\r':
+                    elif c == ord(u'\n') or c == ord(u'\r'):
                         # end of line
                         self.save_field(field_builder)
                         state = EAT_CRNL
@@ -186,10 +183,10 @@
                     else:
                         # illegal
                         raise self.error(u"'%s' expected after '%s'" % (
-                            dialect.delimiter, dialect.quotechar))
+                            unichr(dialect.delimiter), unichr(dialect.quotechar)))
 
                 elif state == EAT_CRNL:
-                    if not (c == u'\n' or c == u'\r'):
+                    if not (c == ord(u'\n') or c == ord(u'\r')):
                         raise self.error(u"new-line character seen in unquoted "
                                          u"field - do you need to open the file "
                                          u"in universal-newline mode?")
@@ -198,16 +195,16 @@
                 self.save_field(field_builder)
                 break
             elif state == ESCAPED_CHAR:
-                self.add_char(field_builder, u'\n')
+                self.add_char(field_builder, ord(u'\n'))
                 state = IN_FIELD
             elif state == IN_QUOTED_FIELD:
                 pass
             elif state == ESCAPE_IN_QUOTED_FIELD:
-                self.add_char(field_builder, u'\n')
+                self.add_char(field_builder, ord(u'\n'))
                 state = IN_QUOTED_FIELD
             elif state == START_FIELD:
                 # save empty field
-                field_builder = UnicodeBuilder(1)
+                field_builder = Utf8StringBuilder()
                 self.save_field(field_builder)
                 break
             elif state == AFTER_ESCAPED_CRNL:
@@ -216,6 +213,8 @@
                 break
         #
         w_result = space.newlist(self.fields_w)
+        # assume all lines have the same number of fields
+        self.sizehint = len(self.fields_w)
         self.fields_w = None
         return w_result
 
diff --git a/pypy/module/_csv/interp_writer.py b/pypy/module/_csv/interp_writer.py
--- a/pypy/module/_csv/interp_writer.py
+++ b/pypy/module/_csv/interp_writer.py
@@ -1,4 +1,4 @@
-from rpython.rlib.rstring import UnicodeBuilder
+from rpython.rlib.rutf8 import Utf8StringIterator, Utf8StringBuilder
 from rpython.rlib import objectmodel
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError
@@ -15,11 +15,13 @@
         self.dialect = dialect
         self.w_filewrite = space.getattr(w_fileobj, space.newtext('write'))
         # precompute this
-        special = dialect.delimiter + dialect.lineterminator
-        if dialect.escapechar != '\0':
-            special += dialect.escapechar
-        if dialect.quotechar != '\0':
-            special += dialect.quotechar
+        special = [dialect.delimiter]
+        for c in Utf8StringIterator(dialect.lineterminator):
+            special.append(c)
+        if dialect.escapechar != 0:
+            special.append(dialect.escapechar)
+        if dialect.quotechar != 0:
+            special.append(dialect.quotechar)
         self.special_characters = special
 
     @objectmodel.dont_inline
@@ -35,16 +37,17 @@
         space = self.space
         fields_w = space.listview(w_fields)
         dialect = self.dialect
-        rec = UnicodeBuilder(80)
+        rec = Utf8StringBuilder(80)
         #
         for field_index in range(len(fields_w)):
             w_field = fields_w[field_index]
             if space.is_w(w_field, space.w_None):
-                field = u""
+                field = ""
+                length = 0
             elif space.isinstance_w(w_field, space.w_float):
-                field = space.realunicode_w(space.repr(w_field))
+                field, length = space.utf8_len_w(space.repr(w_field))
             else:
-                field = space.realunicode_w(space.str(w_field))
+                field, length = space.utf8_len_w(space.str(w_field))
             #
             if dialect.quoting == QUOTE_NONNUMERIC:
                 try:
@@ -57,9 +60,9 @@
             elif dialect.quoting == QUOTE_ALL:
                 quoted = True
             elif dialect.quoting == QUOTE_MINIMAL:
-                # Find out if we really quoting
+                # Find out if we really need quoting.
                 special_characters = self.special_characters
-                for c in field:
+                for c in Utf8StringIterator(field):
                     if c in special_characters:
                         if c != dialect.quotechar or dialect.doublequote:
                             quoted = True
@@ -78,15 +81,15 @@
 
             # If this is not the first field we need a field separator
             if field_index > 0:
-                rec.append(dialect.delimiter)
+                rec.append_code(dialect.delimiter)
 
             # Handle preceding quote
             if quoted:
-                rec.append(dialect.quotechar)
+                rec.append_code(dialect.quotechar)
 
             # Copy field data
             special_characters = self.special_characters
-            for c in field:
+            for c in Utf8StringIterator(field):
                 if c in special_characters:
                     if dialect.quoting == QUOTE_NONE:
                         want_escape = True
@@ -94,28 +97,28 @@
                         want_escape = False
                         if c == dialect.quotechar:
                             if dialect.doublequote:
-                                rec.append(dialect.quotechar)
+                                rec.append_code(dialect.quotechar)
                             else:
                                 want_escape = True
                     if want_escape:
-                        if dialect.escapechar == u'\0':
+                        if dialect.escapechar == 0:
                             raise self.error("need to escape, "
                                              "but no escapechar set")
-                        rec.append(dialect.escapechar)
+                        rec.append_code(dialect.escapechar)
                     else:
                         assert quoted
                 # Copy field character into record buffer
-                rec.append(c)
+                rec.append_code(c)
 
             # Handle final quote
             if quoted:
-                rec.append(dialect.quotechar)
+                rec.append_code(dialect.quotechar)
 
         # Add line terminator
         rec.append(dialect.lineterminator)
 
         line = rec.build()
-        return space.call_function(self.w_filewrite, space.newtext(line))
+        return space.call_function(self.w_filewrite, space.newutf8(line, rec.getlength()))
 
     def writerows(self, w_seqseq):
         """Construct and write a series of sequences to a csv file.
diff --git a/pypy/module/_multibytecodec/src/cjkcodecs/_codecs_cn.c b/pypy/module/_multibytecodec/src/cjkcodecs/_codecs_cn.c
--- a/pypy/module/_multibytecodec/src/cjkcodecs/_codecs_cn.c
+++ b/pypy/module/_multibytecodec/src/cjkcodecs/_codecs_cn.c
@@ -266,7 +266,9 @@
             REQUIRE_INBUF(4)
             c3 = IN3;
             c4 = IN4;
-            if (c < 0x81 || c3 < 0x81 || c4 < 0x30 || c4 > 0x39)
+            if (c  < 0x81 || c  > 0xFE ||
+                c3 < 0x81 || c3 > 0xFE ||
+                c4 < 0x30 || c4 > 0x39)
                 return 1;
             c -= 0x81;  c2 -= 0x30;
             c3 -= 0x81; c4 -= 0x30;
@@ -333,14 +335,16 @@
         DBCHAR code;
 
         if (c < 0x80) {
-            if (state->i == 0) {
-                WRITE1((unsigned char)c)
-                NEXT(1, 1)
+            if (state->i) {
+                WRITE2('~', '}');
+                NEXT_OUT(2);
+                state->i = 0;
             }
-            else {
-                WRITE3('~', '}', (unsigned char)c)
-                NEXT(1, 3)
-                state->i = 0;
+            WRITE1((unsigned char)c);
+            NEXT(1, 1);
+            if (c == '~') {
+                WRITE1('~');
+                NEXT_OUT(1);
             }
             continue;
         }
@@ -388,17 +392,15 @@
             unsigned char c2 = IN2;
 
             REQUIRE_INBUF(2)
-            if (c2 == '~') {
+            if (c2 == '~' && state->i == 0) {
                 WRITE1('~')
-                NEXT(2, 1)
-                continue;
             }
             else if (c2 == '{' && state->i == 0)
                 state->i = 1; /* set GB */
+            else if (c2 == '\n' && state->i == 0)
+                ; /* line-continuation */
             else if (c2 == '}' && state->i == 1)
                 state->i = 0; /* set ASCII */
-            else if (c2 == '\n')
-                ; /* line-continuation */
             else
                 return 1;
             NEXT(2, 0);
diff --git a/pypy/module/_multibytecodec/test/test_c_codecs.py b/pypy/module/_multibytecodec/test/test_c_codecs.py
--- a/pypy/module/_multibytecodec/test/test_c_codecs.py
+++ b/pypy/module/_multibytecodec/test/test_c_codecs.py
@@ -1,4 +1,5 @@
 import py
+import pytest
 from pypy.module._multibytecodec.c_codecs import getcodec, codecs
 from pypy.module._multibytecodec.c_codecs import decode, encode
 from pypy.module._multibytecodec.c_codecs import EncodeDecodeError
@@ -18,6 +19,15 @@
     u = decode(c, "foobar")
     assert u == "foobar"
 
+ at pytest.mark.parametrize('undecodable', [
+    b"abc\x80\x80\xc1\xc4",
+    b"\xff\x30\x81\x30", b"\x81\x30\xff\x30",  # bpo-29990
+])
+def test_decode_gb18030_error(undecodable):
+    c = getcodec("gb18030")
+    with pytest.raises(EncodeDecodeError):
+        decode(c, undecodable)
+
 def test_decode_hz():
     # stateful
     c = getcodec("hz")
@@ -99,6 +109,9 @@
     assert s == 'foobar' and type(s) is str
     s = encode(c, u'\u5f95\u6cef'.encode('utf8'), 2)
     assert s == '~{abc}~}'
+    # bpo-30003
+    s = encode(c, 'ab~cd', 5)
+    assert s == 'ab~~cd'
 
 def test_encode_hz_error():
     # error
diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -73,6 +73,9 @@
     # hit in the cache
     STRING_CACHE_USEFULNESS_FACTOR = 4
 
+    # don't make arbitrarily huge maps
+    MAX_MAP_SIZE = 100
+
 
     def __init__(self, space, s):
         self.space = space
@@ -369,7 +372,7 @@
                 return w_res
             elif ch == ',':
                 i = self.skip_whitespace(i)
-                if currmap.is_state_blocked():
+                if currmap.is_state_blocked() or nextindex > self.MAX_MAP_SIZE:
                     self.scratch.append(values_w)  # can reuse next time
                     dict_w = self._switch_to_dict(currmap, values_w, nextindex)
                     return self.decode_object_dict(i, start, dict_w)
diff --git a/pypy/module/_pypyjson/simd.py b/pypy/module/_pypyjson/simd.py
--- a/pypy/module/_pypyjson/simd.py
+++ b/pypy/module/_pypyjson/simd.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib import objectmodel, unroll
 from rpython.rlib.rarithmetic import r_uint, intmask, LONG_BIT
-from rpython.jit.backend.detect_cpu import autodetect
+from rpython.jit.backend.detect_cpu import autodetect, ProcessorAutodetectError
 
 # accelerators for string operations using simd on regular word sizes (*not*
 # SSE instructions). this style is sometimes called SWAR (SIMD Within A
@@ -15,8 +15,11 @@
     WORD_SIZE = 8
     EVERY_BYTE_ONE = 0x0101010101010101
     EVERY_BYTE_HIGHEST_BIT = 0x8080808080808080
-    if autodetect() == "x86-64":
-        USE_SIMD = True
+    try:
+        if autodetect() == "x86-64":
+            USE_SIMD = True
+    except ProcessorAutodetectError:
+        pass
 else:
     WORD_SIZE = 4
     EVERY_BYTE_ONE = 0x01010101
diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py
--- a/pypy/module/_pypyjson/test/test__pypyjson.py
+++ b/pypy/module/_pypyjson/test/test__pypyjson.py
@@ -467,6 +467,14 @@
         res = _pypyjson.loads(json)
         assert res == [{u'a': 1}, {u'a': 2}]
 
+    def test_huge_map(self):
+        import _pypyjson
+        import __pypy__
+        s = '{' + ",".join('"%s": %s' % (i, i) for i in range(200)) + '}'
+        res = _pypyjson.loads(s)
+        assert len(res) == 200
+        assert __pypy__.strategy(res) == "UnicodeDictStrategy"
+
     def test_tab_in_string_should_fail(self):
         import _pypyjson
         # http://json.org/JSON_checker/test/fail25.json
diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -9,15 +9,15 @@
 # ____________________________________________________________
 
 
-_get_code = lambda frame, w_inputvalue, operr: frame.pycode
+_get_code = lambda frame, w_arg_or_err: frame.pycode
 _decorator = rvmprof.vmprof_execute_code("pypy", _get_code, W_Root)
 my_execute_frame = _decorator(PyFrame.execute_frame)
 
 
 class __extend__(PyFrame):
-    def execute_frame(self, in_generator=None, w_arg_or_err=None):
+    def execute_frame(self, w_arg_or_err=None):
         # indirection for the optional arguments
-        return my_execute_frame(self, in_generator, w_arg_or_err)
+        return my_execute_frame(self, w_arg_or_err)
 
 
 def _safe(s):
diff --git a/pypy/module/_weakref/interp__weakref.py b/pypy/module/_weakref/interp__weakref.py
--- a/pypy/module/_weakref/interp__weakref.py
+++ b/pypy/module/_weakref/interp__weakref.py
@@ -157,6 +157,8 @@
 
 
 class W_WeakrefBase(W_Root):
+    exact_class_applevel_name = 'weakref-or-proxy'
+
     def __init__(self, space, w_obj, w_callable):
         assert w_callable is not space.w_None    # should be really None
         self.space = space
diff --git a/pypy/module/_weakref/test/test_weakref.py b/pypy/module/_weakref/test/test_weakref.py
--- a/pypy/module/_weakref/test/test_weakref.py
+++ b/pypy/module/_weakref/test/test_weakref.py
@@ -545,3 +545,12 @@
         p1[42] = p2
         assert a1.setkey == 42
         assert a1.setvalue is p2
+
+    def test_error_message_wrong_self(self):
+        import _weakref
+        unboundmeth = _weakref.ref.__repr__
+        e = raises(TypeError, unboundmeth, 42)
+        assert "weakref" in str(e.value)
+        if hasattr(unboundmeth, 'im_func'):
+            e = raises(TypeError, unboundmeth.im_func, 42)
+            assert "'weakref-or-proxy'" in str(e.value)
diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -80,6 +80,16 @@
     # XXX this is wrong with IntMutableCell.  Hope it works...
     return w_dict.getitem(w_key)
 
+ at cpython_api([PyObject, PyObject], PyObject, result_borrowed=True)
+def PyDict_GetItemWithError(space, w_dict, w_key):
+    """Variant of PyDict_GetItem() that does not suppress
+    exceptions. Return NULL with an exception set if an exception
+    occurred.  Return NULL without an exception set if the key
+    wasn't present."""
+    if not isinstance(w_dict, W_DictMultiObject):
+        PyErr_BadInternalCall(space)
+    return w_dict.getitem(w_key)
+
 @cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)
 def PyDict_SetItem(space, w_dict, w_key, w_obj):
     if not isinstance(w_dict, W_DictMultiObject):
@@ -298,3 +308,5 @@
             return 0
     return 1
 
+#def PyObject_GenericGetDict(space, w_obj, context):
+#    unlike CPython, you'll find this one in object.py together with ..SetDict
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -444,3 +444,16 @@
             del d[w_obj]
         except KeyError:
             pass
+
+ at cpython_api([PyObject, rffi.VOIDP], PyObject)
+def PyObject_GenericGetDict(space, w_obj, context):
+    from pypy.interpreter.typedef import descr_get_dict
+    return descr_get_dict(space, w_obj)
+
+ at cpython_api([PyObject, PyObject, rffi.VOIDP], rffi.INT_real, error=-1)
+def PyObject_GenericSetDict(space, w_obj, w_value, context):
+    from pypy.interpreter.typedef import descr_set_dict
+    if w_value is None:
+        raise oefmt(space.w_TypeError, "cannot delete __dict__")
+    descr_set_dict(space, w_obj, w_value)
+    return 0
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -243,14 +243,6 @@
 def PyWrapper_New(space, w_d, w_self):
     raise NotImplementedError
 
- at cpython_api([PyObject, PyObject], PyObject)
-def PyDict_GetItemWithError(space, p, key):
-    """Variant of PyDict_GetItem() that does not suppress
-    exceptions. Return NULL with an exception set if an exception
-    occurred.  Return NULL without an exception set if the key
-    wasn't present."""
-    raise NotImplementedError
-
 @cpython_api([PyObject, PyObject, rffi.INT_real], rffi.INT_real, error=-1)
 def PyDict_MergeFromSeq2(space, a, seq2, override):
     """Update or merge into dictionary a, from the key-value pairs in seq2.
diff --git a/pypy/module/cpyext/test/test_dictobject.py b/pypy/module/cpyext/test/test_dictobject.py
--- a/pypy/module/cpyext/test/test_dictobject.py
+++ b/pypy/module/cpyext/test/test_dictobject.py
@@ -175,6 +175,26 @@
             ])
         assert module.dict_proxy({'a': 1, 'b': 2}) == 2
 
+    def test_getitemwitherror(self):
+        module = self.import_extension('foo', [
+            ("dict_getitem", "METH_VARARGS",
+             """
+             PyObject *d, *key, *result;
+             if (!PyArg_ParseTuple(args, "OO", &d, &key)) {
+                return NULL;
+             }
+             result = PyDict_GetItemWithError(d, key);
+             if (result == NULL && !PyErr_Occurred())
+                Py_RETURN_NONE;
+             Py_XINCREF(result);
+             return result;
+             """)])
+        d = {'foo': 'bar'}
+        assert module.dict_getitem(d, 'foo') == 'bar'
+        assert module.dict_getitem(d, 'missing') is None
+        with raises(TypeError):
+            module.dict_getitem(d, [])
+
     def test_setdefault(self):
         module = self.import_extension('foo', [
             ("setdefault", "METH_VARARGS",
diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py
--- a/pypy/module/cpyext/test/test_object.py
+++ b/pypy/module/cpyext/test/test_object.py
@@ -451,6 +451,31 @@
         assert n == 1
         module.leave(obj2)
 
+    def test_GenericGetSetDict(self):
+        module = self.import_extension('test_GenericGetSetDict', [
+            ('test1', 'METH_VARARGS',
+             """
+                 PyObject *obj = PyTuple_GET_ITEM(args, 0);
+                 PyObject *newdict = PyTuple_GET_ITEM(args, 1);
+
+                 PyObject *olddict = PyObject_GenericGetDict(obj, NULL);
+                 if (olddict == NULL)
+                    return NULL;
+                 int res = PyObject_GenericSetDict(obj, newdict, NULL);
+                 if (res != 0)
+                     return NULL;
+                 return olddict;
+             """)])
+        class A:
+            pass
+        a = A()
+        a.x = 42
+        nd = {'y': 43}
+        d = module.test1(a, nd)
+        assert d == {'x': 42}
+        assert a.y == 43
+        assert a.__dict__ is nd
+
 
 class AppTestPyBuffer_FillInfo(AppTestCpythonExtensionBase):
     """
diff --git a/pypy/module/sys/app.py b/pypy/module/sys/app.py
--- a/pypy/module/sys/app.py
+++ b/pypy/module/sys/app.py
@@ -134,7 +134,6 @@
 
 # This is tested in test_app_main.py
 class sysflags(metaclass=structseqtype):
-
     name = "sys.flags"
 
     debug = structseqfield(0)
@@ -157,29 +156,6 @@
 null__xoptions = {}
 
 
-class asyncgen_hooks(metaclass=structseqtype):
-    name = "asyncgen_hooks"
-
-    firstiter = structseqfield(0)
-    finalizer = structseqfield(1)
-
-# FIXME: make this thread-local
-_current_asyncgen_hooks = asyncgen_hooks((None, None))
-
-def get_asyncgen_hooks():
-    return _current_asyncgen_hooks
-
-_default_arg = object()
-
-def set_asyncgen_hooks(firstiter=_default_arg, finalizer=_default_arg):
-    global _current_asyncgen_hooks
-    if firstiter is _default_arg:
-        firstiter = _current_asyncgen_hooks.firstiter
-    if finalizer is _default_arg:
-        finalizer = _current_asyncgen_hooks.finalizer
-    _current_asyncgen_hooks = asyncgen_hooks((firstiter, finalizer))
-
-
 implementation = SimpleNamespace(
     name='pypy',
     version=sys.version_info,
diff --git a/pypy/module/sys/moduledef.py b/pypy/module/sys/moduledef.py
--- a/pypy/module/sys/moduledef.py
+++ b/pypy/module/sys/moduledef.py
@@ -95,6 +95,8 @@
 
         'get_coroutine_wrapper' : 'vm.get_coroutine_wrapper',
         'set_coroutine_wrapper' : 'vm.set_coroutine_wrapper',
+        'get_asyncgen_hooks'    : 'vm.get_asyncgen_hooks',
+        'set_asyncgen_hooks'    : 'vm.set_asyncgen_hooks',
 
         'is_finalizing'         : 'vm.is_finalizing',
         }
@@ -117,8 +119,6 @@
         'flags'                 : 'app.null_sysflags',
         '_xoptions'             : 'app.null__xoptions',
         'implementation'        : 'app.implementation',
-        'get_asyncgen_hooks'    : 'app.get_asyncgen_hooks',
-        'set_asyncgen_hooks'    : 'app.set_asyncgen_hooks',
 
         # these six attributes are here only during tests;
         # they are removed before translation
@@ -186,7 +186,7 @@
                     if w_file is w_stdout:
                         e.write_unraisable(space, '', w_file)
                     ret = -1
-        return ret 
+        return ret
 
     def _file_is_closed(self, space, w_file):
         try:
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -7,7 +7,7 @@
 
 from pypy.interpreter import gateway
 from pypy.interpreter.error import oefmt
-from pypy.interpreter.gateway import unwrap_spec
+from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
 
 
 # ____________________________________________________________
@@ -240,6 +240,14 @@
     suite_mask = structseqfield(12, "Bit mask identifying available product suites")
     product_type = structseqfield(13, "System product type")
     platform_version = structseqfield(14, "Diagnostic version number")
+
+
+class asyncgen_hooks(metaclass=structseqtype):
+    name = "asyncgen_hooks"
+
+    firstiter = structseqfield(0)
+    finalizer = structseqfield(1)
+
 ''')
 
 
@@ -286,7 +294,7 @@
 
 getsizeof_missing = """getsizeof(...)
     getsizeof(object, default) -> int
-    
+
     Return the size of object in bytes.
 
 sys.getsizeof(object, default) will always return default on PyPy, and
@@ -351,5 +359,45 @@
     else:
         raise oefmt(space.w_TypeError, "callable expected, got %T", w_wrapper)
 
+def get_asyncgen_hooks(space):
+    """get_asyncgen_hooks()
+
+Return a namedtuple of installed asynchronous generators hooks (firstiter, finalizer)."""
+    ec = space.getexecutioncontext()
+    w_firstiter = ec.w_asyncgen_firstiter_fn
+    if w_firstiter is None:
+        w_firstiter = space.w_None
+    w_finalizer = ec.w_asyncgen_finalizer_fn
+    if w_finalizer is None:
+        w_finalizer = space.w_None
+    w_asyncgen_hooks = app.wget(space, "asyncgen_hooks")
+    return space.call_function(
+        w_asyncgen_hooks,
+        space.newtuple([w_firstiter, w_finalizer]))
+
+# Note: the docstring is wrong on CPython
+def set_asyncgen_hooks(space, w_firstiter=None, w_finalizer=None):
+    """set_asyncgen_hooks(firstiter=None, finalizer=None)
+
+Set a finalizer for async generators objects."""
+    ec = space.getexecutioncontext()
+    if space.is_w(w_finalizer, space.w_None):
+        ec.w_asyncgen_finalizer_fn = None
+    elif w_finalizer is not None:
+        if space.callable_w(w_finalizer):
+            ec.w_asyncgen_finalizer_fn = w_finalizer
+        else:
+            raise oefmt(space.w_TypeError,
+                "callable finalizer expected, got %T", w_finalizer)
+    if space.is_w(w_firstiter, space.w_None):
+        ec.w_asyncgen_firstiter_fn = None
+    elif w_firstiter is not None:
+        if space.callable_w(w_firstiter):
+            ec.w_asyncgen_firstiter_fn = w_firstiter
+        else:
+            raise oefmt(space.w_TypeError,
+                "callable firstiter expected, got %T", w_firstiter)
+
+
 def is_finalizing(space):
     return space.newbool(space.sys.finalizing)
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -164,6 +164,9 @@
         is_root(w_obj)
         return NonConstant(False)
 
+    def utf8_len_w(self, space):
+        return NonConstant((NonConstant("utf8len_foobar"), NonConstant(14)))
+
     @not_rpython
     def unwrap(self, w_obj):
         raise NotImplementedError
diff --git a/pypy/objspace/fake/test/test_objspace.py b/pypy/objspace/fake/test/test_objspace.py
--- a/pypy/objspace/fake/test/test_objspace.py
+++ b/pypy/objspace/fake/test/test_objspace.py
@@ -1,4 +1,5 @@
-import py
+import pytest
+from rpython.rlib.nonconst import NonConstant
 from pypy.objspace.fake.objspace import FakeObjSpace, W_Root
 from pypy.interpreter.argument import Arguments
 from pypy.interpreter.typedef import TypeDef
@@ -63,8 +64,8 @@
     def test_is_true(self):
         space = self.space
         space.translates(lambda: space.is_true(W_Root()))
-        py.test.raises(AssertionError,
-                       space.translates, lambda: space.is_true(42))
+        with pytest.raises(AssertionError):
+            space.translates(lambda: space.is_true(42))
 
     def test_unpackiterable(self):
         space = self.space
@@ -79,3 +80,23 @@
         space = self.space
         space.translates(lambda: (space.get(W_Root(), W_Root()),
                                   space.get(W_Root(), W_Root(), W_Root())))
+
+    def test_bug_utf8_len_w(self):
+        space = self.space
+
+        class A(object):
+            pass
+
+        def f():
+            s = NonConstant('a')
+            w_s = space.newutf8(s, 1)
+            t, l = space.utf8_len_w(w_s)
+            a = A()
+            if l == 1:
+                a.x = 1
+            else:
+                raise Exception
+            return a.x
+        space.translates(f)
+
+
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -19,6 +19,7 @@
 
 class W_AbstractBytesObject(W_Root):
     __slots__ = ()
+    exact_class_applevel_name = 'bytes'
 
     def is_w(self, space, w_other):
         if not isinstance(w_other, W_AbstractBytesObject):
diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -20,6 +20,7 @@
 
 class W_BaseSetObject(W_Root):
     typedef = None
+    exact_class_applevel_name = 'set-or-frozenset'
 
     def __init__(self, space, w_iterable=None):
         """Initialize the set by taking ownership of 'setdata'."""
@@ -489,6 +490,12 @@
 
 
 class W_SetObject(W_BaseSetObject):
+
+    #overridden here so the error is reported correctly
+    def __init__(self, space, w_iterable=None):
+        """Initialize the set by taking ownership of 'setdata'."""
+        W_BaseSetObject.__init__(self, space, w_iterable)
+
     def _newobj(self, space, w_iterable):
         """Make a new set by taking ownership of 'w_iterable'."""
         return W_SetObject(space, w_iterable)
@@ -504,7 +511,7 @@
 
 Build an unordered collection.""",
     __new__ = gateway.interp2app(W_SetObject.descr_new),
-    __init__ = gateway.interp2app(W_BaseSetObject.descr_init),
+    __init__ = gateway.interp2app(W_SetObject.descr_init),
     __repr__ = gateway.interp2app(W_BaseSetObject.descr_repr),
     __hash__ = None,
 
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -1050,3 +1050,10 @@
         id_a = id(a)
         assert a is not str(a, 'latin1')
         assert id_a != id_b
+
+    def test_error_message_wrong_self(self):
+        e = raises(TypeError, bytes.upper, 42)
+        assert "bytes" in str(e.value)
+        if hasattr(bytes.upper, 'im_func'):
+            e = raises(TypeError, bytes.upper.im_func, 42)
+            assert "'bytes'" in str(e.value)

From pypy.commits at gmail.com  Wed Oct 16 14:21:40 2019
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 16 Oct 2019 11:21:40 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: reject null characters in a few more
 functions (bpo-13617)
Message-ID: <5da75fb4.1c69fb81.4581e.129c@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6
Changeset: r97800:24c1cf8d6d14
Date: 2019-10-16 19:20 +0100
http://bitbucket.org/pypy/pypy/changeset/24c1cf8d6d14/

Log:	reject null characters in a few more functions (bpo-13617)

diff --git a/lib_pypy/grp.py b/lib_pypy/grp.py
--- a/lib_pypy/grp.py
+++ b/lib_pypy/grp.py
@@ -47,12 +47,16 @@
             # XXX maybe check error eventually
             raise KeyError(gid)
         return _group_from_gstruct(res)
+
 @builtinify
 def getgrnam(name):
     if not isinstance(name, str):
         raise TypeError("expected string")
+    name_b = os.fsencode(name)
+    if b'\0' in name_b:
+        raise ValueError("embedded null byte")
     with _lock:
-        res = lib.getgrnam(os.fsencode(name))
+        res = lib.getgrnam(name_b)
         if not res:
             raise KeyError("getgrnam(): name not found: %s" % name)
         return _group_from_gstruct(res)
diff --git a/pypy/module/_locale/interp_locale.py b/pypy/module/_locale/interp_locale.py
--- a/pypy/module/_locale/interp_locale.py
+++ b/pypy/module/_locale/interp_locale.py
@@ -98,6 +98,8 @@
 
     s1, l1 = space.utf8_len_w(w_s1)
     s2, l2 = space.utf8_len_w(w_s2)
+    if '\x00' in s1 or '\x00' in s2:
+        raise oefmt(space.w_ValueError, "embedded null character")
 
     s1_c = rffi.utf82wcharp(s1, l1)
     s2_c = rffi.utf82wcharp(s2, l2)
@@ -112,7 +114,7 @@
 _strxfrm = rlocale.external('strxfrm',
                     [rffi.CCHARP, rffi.CCHARP, rffi.SIZE_T], rffi.SIZE_T)
 
- at unwrap_spec(s='text')
+ at unwrap_spec(s='text0')
 def strxfrm(space, s):
     "string -> string. Returns a string that behaves for cmp locale-aware."
     n1 = len(s) + 1
diff --git a/pypy/module/_locale/test/test_locale.py b/pypy/module/_locale/test/test_locale.py
--- a/pypy/module/_locale/test/test_locale.py
+++ b/pypy/module/_locale/test/test_locale.py
@@ -49,8 +49,8 @@
     def teardown_class(cls):
         import _locale
         _locale.setlocale(_locale.LC_ALL, cls.oldlocale)
-        
-        
+
+
 
     def test_import(self):
         import _locale
@@ -58,7 +58,7 @@
 
         import locale
         assert locale
-        
+
     def test_constants(self):
         import sys
 
@@ -82,7 +82,7 @@
         )
 
         import _locale
-        
+
         for constant in _CONSTANTS:
             assert hasattr(_locale, constant)
 
@@ -171,7 +171,10 @@
         assert a is not b
         assert a == b
 
-        raises(TypeError, _locale.strxfrm, 1)
+        with raises(TypeError):
+            _locale.strxfrm(1)
+        with raises(ValueError):
+            _locale.strxfrm("a\x00b")
 
         _locale.setlocale(_locale.LC_ALL, self.language_pl)
         a = "1234"
@@ -271,7 +274,7 @@
 
         raises(ValueError, _locale.nl_langinfo, 12345)
         raises(TypeError, _locale.nl_langinfo, None)
-    
+
     def test_bindtextdomain(self):
         import sys
         if sys.platform == 'win32':
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -837,7 +837,7 @@
         # reset timezone, altzone, daylight and tzname
         _init_timezone(space)
 
- at unwrap_spec(format='text')
+ at unwrap_spec(format='text0')
 def strftime(space, format, w_tup=None):
     """strftime(format[, tuple]) -> string
 
diff --git a/pypy/module/time/test/test_time.py b/pypy/module/time/test/test_time.py
--- a/pypy/module/time/test/test_time.py
+++ b/pypy/module/time/test/test_time.py
@@ -315,6 +315,8 @@
         else:
             assert time.strftime('%f') == '%f'
             expected_year = '0'
+        with raises(ValueError):
+            time.strftime("%Y\0", tt)
 
         expected_formatted_date = expected_year + ' 01 01 00 00 00 1 001'
         assert time.strftime("%Y %m %d %H %M %S %w %j", (0,) * 9) == expected_formatted_date

From pypy.commits at gmail.com  Thu Oct 17 13:29:01 2019
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 17 Oct 2019 10:29:01 -0700 (PDT)
Subject: [pypy-commit] pypy default: Correctly swallow exceptions happening
 inside PyDict_GetItem() (#3098)
Message-ID: <5da8a4dd.1c69fb81.c9242.f973@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r97801:183ffc0a3903
Date: 2019-10-17 18:26 +0100
http://bitbucket.org/pypy/pypy/changeset/183ffc0a3903/

Log:	Correctly swallow exceptions happening inside PyDict_GetItem()
	(#3098)

diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -78,7 +78,10 @@
     # *values* as full objects, which stay alive as long as the dict is
     # alive and not modified.  So we can return a borrowed ref.
     # XXX this is wrong with IntMutableCell.  Hope it works...
-    return w_dict.getitem(w_key)
+    try:
+        return w_dict.getitem(w_key)
+    except OperationError:
+        return None
 
 @cpython_api([PyObject, PyObject], PyObject, result_borrowed=True)
 def _PyDict_GetItemWithError(space, w_dict, w_key):
diff --git a/pypy/module/cpyext/test/test_dictobject.py b/pypy/module/cpyext/test/test_dictobject.py
--- a/pypy/module/cpyext/test/test_dictobject.py
+++ b/pypy/module/cpyext/test/test_dictobject.py
@@ -394,3 +394,20 @@
         d[1] = 2
         assert d[1] == 42
         assert module.dict_getitem(d, 1) == 2
+
+    def test_getitem_error(self):
+        module = self.import_extension('foo', [
+            ("dict_getitem", "METH_VARARGS",
+             """
+             PyObject *d, *key, *result;
+             if (!PyArg_ParseTuple(args, "OO", &d, &key)) {
+                return NULL;
+             }
+             result = PyDict_GetItem(d, key);
+             if (!result) Py_RETURN_NONE;
+             Py_XINCREF(result);
+             return result;
+             """),
+        ])
+        assert module.dict_getitem(42, 43) is None
+        assert module.dict_getitem({}, []) is None

From pypy.commits at gmail.com  Thu Oct 17 13:29:03 2019
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 17 Oct 2019 10:29:03 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: hg merge default
Message-ID: <5da8a4df.1c69fb81.5591c.df66@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6
Changeset: r97802:2126ee5b7acd
Date: 2019-10-17 18:28 +0100
http://bitbucket.org/pypy/pypy/changeset/2126ee5b7acd/

Log:	hg merge default

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -22,9 +22,13 @@
 
 __all__ = ['ObjSpace', 'OperationError', 'W_Root']
 
+def get_printable_location(tp):
+    return "unpackiterable: %s" % (tp, )
+
 unpackiterable_driver = jit.JitDriver(name='unpackiterable',
                                       greens=['tp'],
-                                      reds=['items', 'w_iterator'])
+                                      reds=['items', 'w_iterator'],
+                                      get_printable_location=get_printable_location)
 
 
 class W_Root(object):
diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -78,7 +78,10 @@
     # *values* as full objects, which stay alive as long as the dict is
     # alive and not modified.  So we can return a borrowed ref.
     # XXX this is wrong with IntMutableCell.  Hope it works...
-    return w_dict.getitem(w_key)
+    try:
+        return w_dict.getitem(w_key)
+    except OperationError:
+        return None
 
 @cpython_api([PyObject, PyObject], PyObject, result_borrowed=True)
 def PyDict_GetItemWithError(space, w_dict, w_key):
diff --git a/pypy/module/cpyext/test/test_dictobject.py b/pypy/module/cpyext/test/test_dictobject.py
--- a/pypy/module/cpyext/test/test_dictobject.py
+++ b/pypy/module/cpyext/test/test_dictobject.py
@@ -416,3 +416,20 @@
         d[1] = 2
         assert d[1] == 42
         assert module.dict_getitem(d, 1) == 2
+
+    def test_getitem_error(self):
+        module = self.import_extension('foo', [
+            ("dict_getitem", "METH_VARARGS",
+             """
+             PyObject *d, *key, *result;
+             if (!PyArg_ParseTuple(args, "OO", &d, &key)) {
+                return NULL;
+             }
+             result = PyDict_GetItem(d, key);
+             if (!result) Py_RETURN_NONE;
+             Py_XINCREF(result);
+             return result;
+             """),
+        ])
+        assert module.dict_getitem(42, 43) is None
+        assert module.dict_getitem({}, []) is None
diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -693,10 +693,12 @@
             return unerase_item(erased)
 
         def _mapdict_write_storage(self, storageindex, value):
-            for i in rangenmin1:
-                if storageindex == i:
-                    setattr(self, "_value%s" % i, value)
-                    return
+            assert storageindex >= 0
+            if storageindex < nmin1:
+                for i in rangenmin1:
+                    if storageindex == i:
+                        setattr(self, "_value%s" % i, value)
+                        return
             if self._has_storage_list():
                 self._mapdict_get_storage_list()[storageindex - nmin1] = value
                 return
diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -1658,9 +1658,13 @@
     w_set.sstorage = w_set.strategy.get_storage_from_list(iterable_w)
 
 
+def get_printable_location(tp, strategy):
+    return "create_set: %s %s" % (tp, strategy)
+
 create_set_driver = jit.JitDriver(name='create_set',
                                   greens=['tp', 'strategy'],
-                                  reds='auto')
+                                  reds='auto',
+                                  get_printable_location=get_printable_location)
 
 def _create_from_iterable(space, w_set, w_iterable):
     w_set.strategy = strategy = space.fromcache(EmptySetStrategy)

From pypy.commits at gmail.com  Thu Oct 17 13:54:07 2019
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 17 Oct 2019 10:54:07 -0700 (PDT)
Subject: [pypy-commit] pypy default: Avoid invalid escapes in strings: they
 cause SyntaxWarnings in py3.6 (#3097)
Message-ID: <5da8aabf.1c69fb81.b64ce.db4b@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r97803:099433b77539
Date: 2019-10-17 18:53 +0100
http://bitbucket.org/pypy/pypy/changeset/099433b77539/

Log:	Avoid invalid escapes in strings: they cause SyntaxWarnings in py3.6
	(#3097)

diff --git a/lib_pypy/pyrepl/historical_reader.py b/lib_pypy/pyrepl/historical_reader.py
--- a/lib_pypy/pyrepl/historical_reader.py
+++ b/lib_pypy/pyrepl/historical_reader.py
@@ -111,7 +111,7 @@
         r.isearch_term = ''
         r.dirty = 1
         r.push_input_trans(r.isearch_trans)
-        
+
 
 class reverse_history_isearch(commands.Command):
     def do(self):
@@ -217,7 +217,7 @@
         self.isearch_trans = input.KeymapTranslator(
             isearch_keymap, invalid_cls=isearch_end,
             character_cls=isearch_add_character)
-        
+
     def select_item(self, i):
         self.transient_history[self.historyi] = self.get_unicode()
         buf = self.transient_history.get(i)
@@ -302,7 +302,7 @@
     reader.ps1 = "h**> "
     reader.ps2 = "h/*> "
     reader.ps3 = "h|*> "
-    reader.ps4 = "h\*> "
+    reader.ps4 = r"h\*> "
     while reader.readline():
         pass
 
diff --git a/lib_pypy/pyrepl/keymap.py b/lib_pypy/pyrepl/keymap.py
--- a/lib_pypy/pyrepl/keymap.py
+++ b/lib_pypy/pyrepl/keymap.py
@@ -58,7 +58,7 @@
     "'":"'",
     '"':'"',
     'a':'\a',
-    'b':'\h',
+    'b':'\b',
     'e':'\033',
     'f':'\f',
     'n':'\n',
diff --git a/lib_pypy/pyrepl/reader.py b/lib_pypy/pyrepl/reader.py
--- a/lib_pypy/pyrepl/reader.py
+++ b/lib_pypy/pyrepl/reader.py
@@ -232,7 +232,7 @@
         self.ps1 = "->> "
         self.ps2 = "/>> "
         self.ps3 = "|.. "
-        self.ps4 = "\__ "
+        self.ps4 = r"\__ "
         self.kill_ring = []
         self.arg = None
         self.finished = 0

From pypy.commits at gmail.com  Thu Oct 17 17:23:06 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 17 Oct 2019 14:23:06 -0700 (PDT)
Subject: [pypy-commit] pypy default: issue #3099: allow reloading of
 subclasses of module
Message-ID: <5da8dbba.1c69fb81.be66d.24d1@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r97805:f27546b858f9
Date: 2019-10-17 23:22 +0200
http://bitbucket.org/pypy/pypy/changeset/f27546b858f9/

Log:	issue #3099: allow reloading of subclasses of module

diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -723,7 +723,7 @@
 def reload(space, w_module):
     """Reload the module.
     The module must have been successfully imported before."""
-    if not space.is_w(space.type(w_module), space.type(space.sys)):
+    if not space.isinstance_w(w_module, space.type(space.sys)):
         raise oefmt(space.w_TypeError, "reload() argument must be module")
 
     w_modulename = space.getattr(w_module, space.newtext("__name__"))
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -656,6 +656,20 @@
     def test_reload_infinite(self):
         import infinite_reload
 
+    def test_reload_module_subclass(self):
+        import types
+
+        #MyModType = types.ModuleType
+        class MyModType(types.ModuleType):
+            pass
+
+        m = MyModType("abc")
+        with raises(ImportError):
+            # Fails because the module is not in sys.modules, but *not* because
+            # it's a subtype of ModuleType.
+            reload(m)
+
+
     def test_explicitly_missing(self):
         import sys
         sys.modules['foobarbazmod'] = None

From pypy.commits at gmail.com  Fri Oct 18 02:30:16 2019
From: pypy.commits at gmail.com (arigo)
Date: Thu, 17 Oct 2019 23:30:16 -0700 (PDT)
Subject: [pypy-commit] pypy default: Arguably,
 clarify the logic.  The real motivation is a gcc bug, see issue #3086
Message-ID: <5da95bf8.1c69fb81.328f7.f0e1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r97806:d81c769a2353
Date: 2019-10-18 08:29 +0200
http://bitbucket.org/pypy/pypy/changeset/d81c769a2353/

Log:	Arguably, clarify the logic. The real motivation is a gcc bug, see
	issue #3086

diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py
--- a/rpython/jit/backend/aarch64/opassembler.py
+++ b/rpython/jit/backend/aarch64/opassembler.py
@@ -808,9 +808,7 @@
             # Inline a series of STR operations, starting at 'dstaddr_loc'.
             #
             self.mc.gen_load_int(r.ip0.value, 0)
-            i = 0
-            adjustment = 0
-            needs_adjustment = itemsize < 8 and (startbyte % 8)
+            i = dst_i = 0
             total_size = size_box.getint()
             while i < total_size:
                 sz = itemsize
@@ -818,19 +816,19 @@
                     next_group += 8
                     if next_group <= total_size:
                         sz = 8
+                        if dst_i % 8:   # unaligned?
+                            self.mc.ADD_ri(dstaddr_loc.value, dstaddr_loc.value, dst_i)
+                            dst_i = 0
                 if sz == 8:
-                    if needs_adjustment:
-                        self.mc.ADD_ri(dstaddr_loc.value, dstaddr_loc.value, i)
-                        adjustment = -i
-                        needs_adjustment = False
-                    self.mc.STR_ri(r.ip0.value, dstaddr_loc.value, i + adjustment)                    
+                    self.mc.STR_ri(r.ip0.value, dstaddr_loc.value, dst_i)
                 elif sz == 4:
-                    self.mc.STRW_ri(r.ip0.value, dstaddr_loc.value, i + adjustment)
+                    self.mc.STRW_ri(r.ip0.value, dstaddr_loc.value, dst_i)
                 elif sz == 2:
-                    self.mc.STRH_ri(r.ip0.value, dstaddr_loc.value, i + adjustment)
+                    self.mc.STRH_ri(r.ip0.value, dstaddr_loc.value, dst_i)
                 else:
-                    self.mc.STRB_ri(r.ip0.value, dstaddr_loc.value, i + adjustment)
+                    self.mc.STRB_ri(r.ip0.value, dstaddr_loc.value, dst_i)
                 i += sz
+                dst_i += sz
 
         else:
             if isinstance(size_box, ConstInt):

From pypy.commits at gmail.com  Fri Oct 18 03:24:42 2019
From: pypy.commits at gmail.com (mattip)
Date: Fri, 18 Oct 2019 00:24:42 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: macos does not have clock_gettime until
 macOS 10.12 and Xcode 8, released in 2016
Message-ID: <5da968ba.1c69fb81.1d5af.b1f6@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r97807:7f8a8e590485
Date: 2019-10-18 10:23 +0300
http://bitbucket.org/pypy/pypy/changeset/7f8a8e590485/

Log:	macos does not have clock_gettime until macOS 10.12 and Xcode 8,
	released in 2016

diff --git a/pypy/module/cpyext/include/pyconfig.h b/pypy/module/cpyext/include/pyconfig.h
--- a/pypy/module/cpyext/include/pyconfig.h
+++ b/pypy/module/cpyext/include/pyconfig.h
@@ -28,8 +28,10 @@
 
 #ifndef _WIN32
 #define VA_LIST_IS_ARRAY
+#ifndef __APPLE__
 #define HAVE_CLOCK_GETTIME
 #endif
+#endif
 
 #ifndef Py_BUILD_CORE /* not building the core - must be an ext */
 #  if defined(_MSC_VER) && !defined(_CFFI_)

From pypy.commits at gmail.com  Fri Oct 18 04:05:43 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 18 Oct 2019 01:05:43 -0700 (PDT)
Subject: [pypy-commit] pypy record-known-result: fix hint for corner cases,
 add rfind version
Message-ID: <5da97257.1c69fb81.bfbac.bb53@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: record-known-result
Changeset: r97808:7cf364a94ebb
Date: 2019-10-18 10:00 +0200
http://bitbucket.org/pypy/pypy/changeset/7cf364a94ebb/

Log:	fix hint for corner cases, add rfind version

diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -728,14 +728,15 @@
             return -1
 
         m = len(s2.chars)
-        if m == 1:
+        if m <= 1:
+            if m == 0:
+                return start
             res = LLHelpers.ll_find_char(s1, s2.chars[0], start, end)
             jit.record_exact_value(res < end, True)
             return res
 
         res = LLHelpers.ll_search(s1, s2, start, end, FAST_FIND)
         jit.record_exact_value(res < end, True)
-        jit.record_exact_value(res + m <= n, True)
         return res
 
     @staticmethod
@@ -749,10 +750,17 @@
             return -1
 
         m = len(s2.chars)
-        if m == 1:
-            return LLHelpers.ll_rfind_char(s1, s2.chars[0], start, end)
+        if m <= 1:
+            if m == 0:
+                return end
+            LLHelpers.ll_find_char(s1, s2.chars[0], start, end)
+            res = LLHelpers.ll_rfind_char(s1, s2.chars[0], start, end)
+            jit.record_exact_value(res < end, True)
+            return res
 
-        return LLHelpers.ll_search(s1, s2, start, end, FAST_RFIND)
+        res = LLHelpers.ll_search(s1, s2, start, end, FAST_RFIND)
+        jit.record_exact_value(res < end, True)
+        return res
 
     @classmethod
     def ll_count(cls, s1, s2, start, end):

From pypy.commits at gmail.com  Fri Oct 18 09:51:47 2019
From: pypy.commits at gmail.com (mattip)
Date: Fri, 18 Oct 2019 06:51:47 -0700 (PDT)
Subject: [pypy-commit] pypy default: restart whatsnew-pypy3-head.rst (fwiw)
Message-ID: <5da9c373.1c69fb81.5be2e.1841@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r97809:6dac7d2ba98d
Date: 2019-10-18 16:29 +0300
http://bitbucket.org/pypy/pypy/changeset/6dac7d2ba98d/

Log:	restart whatsnew-pypy3-head.rst (fwiw)

diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -1,7 +1,7 @@
-========================
-What's new in PyPy3 7.0+
-========================
-
-.. this is the revision after release-pypy3.5-v7.0
-.. startrev: 9d2fa7c63b7c
-
+========================
+What's new in PyPy3 7.2+
+========================
+
+.. this is the revision after release-pypy3.6-v7.2
+.. startrev: 6d2f8470165b
+

From pypy.commits at gmail.com  Fri Oct 18 09:51:49 2019
From: pypy.commits at gmail.com (mattip)
Date: Fri, 18 Oct 2019 06:51:49 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: merge default into branch
Message-ID: <5da9c375.1c69fb81.e2cd.b9f9@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r97810:99f01005c2c6
Date: 2019-10-18 16:38 +0300
http://bitbucket.org/pypy/pypy/changeset/99f01005c2c6/

Log:	merge default into branch

diff --git a/lib_pypy/pyrepl/historical_reader.py b/lib_pypy/pyrepl/historical_reader.py
--- a/lib_pypy/pyrepl/historical_reader.py
+++ b/lib_pypy/pyrepl/historical_reader.py
@@ -112,7 +112,7 @@
         r.isearch_term = ''
         r.dirty = 1
         r.push_input_trans(r.isearch_trans)
-        
+
 
 class reverse_history_isearch(commands.Command):
     def do(self):
@@ -218,7 +218,7 @@
         self.isearch_trans = input.KeymapTranslator(
             isearch_keymap, invalid_cls=isearch_end,
             character_cls=isearch_add_character)
-        
+
     def select_item(self, i):
         self.transient_history[self.historyi] = self.get_unicode()
         buf = self.transient_history.get(i)
@@ -303,7 +303,7 @@
     reader.ps1 = "h**> "
     reader.ps2 = "h/*> "
     reader.ps3 = "h|*> "
-    reader.ps4 = "h\*> "
+    reader.ps4 = r"h\*> "
     while reader.readline():
         pass
 
diff --git a/lib_pypy/pyrepl/keymap.py b/lib_pypy/pyrepl/keymap.py
--- a/lib_pypy/pyrepl/keymap.py
+++ b/lib_pypy/pyrepl/keymap.py
@@ -58,7 +58,7 @@
     "'":"'",
     '"':'"',
     'a':'\a',
-    'b':'\h',
+    'b':'\b',
     'e':'\033',
     'f':'\f',
     'n':'\n',
diff --git a/lib_pypy/pyrepl/reader.py b/lib_pypy/pyrepl/reader.py
--- a/lib_pypy/pyrepl/reader.py
+++ b/lib_pypy/pyrepl/reader.py
@@ -247,7 +247,7 @@
         self.ps1 = "->> "
         self.ps2 = "/>> "
         self.ps3 = "|.. "
-        self.ps4 = "\__ "
+        self.ps4 = r"\__ "
         self.kill_ring = []
         self.arg = None
         self.finished = 0
diff --git a/pypy/doc/whatsnew-pypy3-5.10.0.rst b/pypy/doc/whatsnew-pypy3-5.10.0.rst
deleted file mode 100644
--- a/pypy/doc/whatsnew-pypy3-5.10.0.rst
+++ /dev/null
@@ -1,7 +0,0 @@
-========================
-What's new in PyPy3 7.0+
-========================
-
-.. this is the revision after release-pypy3.5-v7.0
-.. startrev: 9d2fa7c63b7c
-
diff --git a/pypy/doc/whatsnew-pypy3-6.0.0.rst b/pypy/doc/whatsnew-pypy3-6.0.0.rst
deleted file mode 100644
--- a/pypy/doc/whatsnew-pypy3-6.0.0.rst
+++ /dev/null
@@ -1,7 +0,0 @@
-========================
-What's new in PyPy3 7.0+
-========================
-
-.. this is the revision after release-pypy3.5-v7.0
-.. startrev: 9d2fa7c63b7c
-
diff --git a/pypy/doc/whatsnew-pypy3-7.1.0.rst b/pypy/doc/whatsnew-pypy3-7.1.0.rst
deleted file mode 100644
--- a/pypy/doc/whatsnew-pypy3-7.1.0.rst
+++ /dev/null
@@ -1,11 +0,0 @@
-========================
-What's new in PyPy3 7.0+
-========================
-
-.. this is the revision after release-pypy3.6-v7.0
-.. startrev: 33fe3b2cf186
-
-.. branch: py3.5
-
-Merge in py.35 and use this branch as the primary pypy3 one
-
diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -1,58 +1,11 @@
 ========================
-What's new in PyPy3 7.1+
+What's new in PyPy3 7.2+
 ========================
 
-.. this is the revision after release-pypy3.6-v7.1.1
-.. startrev: db5a1e7fbbd0
+.. this is the revision after release-pypy3.6-v7.2
+.. startrev: 6d2f8470165b
 
-.. branch: fix-literal-prev_digit-underscore
 
-Fix parsing for converting strings with underscore into ints
+.. branch: py3.6-asyncgen
 
-.. branch: winmultiprocessing
-
-Improve multiprocessing support on win32
-
-.. branch: setitem2d
-
-Allow 2d indexing in ``memoryview.__setitem__`` (issue bb-3028)
-
-.. branch: py3.6-socket-fix
-.. branch: fix-importerror
-.. branch: dj_s390
-.. branch: bpo-35409
-.. branch: remove_array_with_char_test
-.. branch: fix_test_unicode_outofrange
-.. branch: Ram-Rachum/faulthandleris_enabled-should-return-fal-1563636614875
-.. branch: Anthony-Sottile/fix-leak-of-file-descriptor-with-_iofile-1559687440863
-
-.. branch: py3tests
-
-Add handling of application-level test files and -D flag to test runner
-
-.. branch: vendor/stdlib-3.6
-.. branch: stdlib-3.6.9
-
-Update standard library to version 3.6.9
-
-.. branch: __debug__-optimize
-
-Fix handling of __debug__, sys.flags.optimize, and '-O' command-line flag to 
-match CPython 3.6.
-
-.. branch: more-cpyext
-
-Add ``PyErr_SetFromWindowsErr`` and ``pytime.h``, ``pytime.c``. Fix order of
-fields in ``Py_buffer``.
-
-.. branch: Ryan-Hileman/add-support-for-zipfile-stdlib-1562420744699
-
-Add support for the entire stdlib being inside a zipfile
-
-
-.. branch: json-decoder-maps-py3.6
-
-Much faster and more memory-efficient JSON decoding. The resulting
-dictionaries that come out of the JSON decoder have faster lookups too.
-
-
+Fix asyncgen_hooks and refactor coroutine execution
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -643,6 +643,20 @@
     def test_reload_infinite(self):
         import infinite_reload
 
+    def test_reload_module_subclass(self):
+        import types
+
+        #MyModType = types.ModuleType
+        class MyModType(types.ModuleType):
+            pass
+
+        m = MyModType("abc")
+        with raises(ImportError):
+            # Fails because the module is not in sys.modules, but *not* because
+            # it's a subtype of ModuleType.
+            reload(m)
+
+
     def test_explicitly_missing(self):
         import sys
         sys.modules['foobarbazmod'] = None
diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py
--- a/rpython/jit/backend/aarch64/opassembler.py
+++ b/rpython/jit/backend/aarch64/opassembler.py
@@ -808,9 +808,7 @@
             # Inline a series of STR operations, starting at 'dstaddr_loc'.
             #
             self.mc.gen_load_int(r.ip0.value, 0)
-            i = 0
-            adjustment = 0
-            needs_adjustment = itemsize < 8 and (startbyte % 8)
+            i = dst_i = 0
             total_size = size_box.getint()
             while i < total_size:
                 sz = itemsize
@@ -818,19 +816,19 @@
                     next_group += 8
                     if next_group <= total_size:
                         sz = 8
+                        if dst_i % 8:   # unaligned?
+                            self.mc.ADD_ri(dstaddr_loc.value, dstaddr_loc.value, dst_i)
+                            dst_i = 0
                 if sz == 8:
-                    if needs_adjustment:
-                        self.mc.ADD_ri(dstaddr_loc.value, dstaddr_loc.value, i)
-                        adjustment = -i
-                        needs_adjustment = False
-                    self.mc.STR_ri(r.ip0.value, dstaddr_loc.value, i + adjustment)                    
+                    self.mc.STR_ri(r.ip0.value, dstaddr_loc.value, dst_i)
                 elif sz == 4:
-                    self.mc.STRW_ri(r.ip0.value, dstaddr_loc.value, i + adjustment)
+                    self.mc.STRW_ri(r.ip0.value, dstaddr_loc.value, dst_i)
                 elif sz == 2:
-                    self.mc.STRH_ri(r.ip0.value, dstaddr_loc.value, i + adjustment)
+                    self.mc.STRH_ri(r.ip0.value, dstaddr_loc.value, dst_i)
                 else:
-                    self.mc.STRB_ri(r.ip0.value, dstaddr_loc.value, i + adjustment)
+                    self.mc.STRB_ri(r.ip0.value, dstaddr_loc.value, dst_i)
                 i += sz
+                dst_i += sz
 
         else:
             if isinstance(size_box, ConstInt):

From pypy.commits at gmail.com  Fri Oct 18 09:51:51 2019
From: pypy.commits at gmail.com (mattip)
Date: Fri, 18 Oct 2019 06:51:51 -0700 (PDT)
Subject: [pypy-commit] pypy default: backport file from py3.6 branch and
 restore missing files
Message-ID: <5da9c377.1c69fb81.edd8e.d0a8@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r97811:ebb5f879d17a
Date: 2019-10-18 16:42 +0300
http://bitbucket.org/pypy/pypy/changeset/ebb5f879d17a/

Log:	backport file from py3.6 branch and restore missing files

diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -43,6 +43,7 @@
 
 .. toctree::
     whatsnew-pypy3-head.rst
+    whatsnew-pypy3-7.2.0.rst
     whatsnew-pypy3-7.1.0.rst
 
 CPython 3.5 compatible versions
diff --git a/pypy/doc/whatsnew-pypy3-5.10.0.rst b/pypy/doc/whatsnew-pypy3-5.10.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy3-5.10.0.rst
@@ -0,0 +1,7 @@
+========================
+What's new in PyPy3 7.0+
+========================
+
+.. this is the revision after release-pypy3.5-v7.0
+.. startrev: 9d2fa7c63b7c
+
diff --git a/pypy/doc/whatsnew-pypy3-6.0.0.rst b/pypy/doc/whatsnew-pypy3-6.0.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy3-6.0.0.rst
@@ -0,0 +1,7 @@
+========================
+What's new in PyPy3 7.0+
+========================
+
+.. this is the revision after release-pypy3.5-v7.0
+.. startrev: 9d2fa7c63b7c
+
diff --git a/pypy/doc/whatsnew-pypy3-7.1.0.rst b/pypy/doc/whatsnew-pypy3-7.1.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy3-7.1.0.rst
@@ -0,0 +1,11 @@
+========================
+What's new in PyPy3 7.0+
+========================
+
+.. this is the revision after release-pypy3.6-v7.0
+.. startrev: 33fe3b2cf186
+
+.. branch: py3.5
+
+Merge in py.35 and use this branch as the primary pypy3 one
+
diff --git a/pypy/doc/whatsnew-pypy3-7.2.0.rst b/pypy/doc/whatsnew-pypy3-7.2.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy3-7.2.0.rst
@@ -0,0 +1,58 @@
+=========================
+What's new in PyPy3 7.2.0
+=========================
+
+.. this is the revision after release-pypy3.6-v7.1.1
+.. startrev: db5a1e7fbbd0
+
+.. branch: fix-literal-prev_digit-underscore
+
+Fix parsing for converting strings with underscore into ints
+
+.. branch: winmultiprocessing
+
+Improve multiprocessing support on win32
+
+.. branch: setitem2d
+
+Allow 2d indexing in ``memoryview.__setitem__`` (issue bb-3028)
+
+.. branch: py3.6-socket-fix
+.. branch: fix-importerror
+.. branch: dj_s390
+.. branch: bpo-35409
+.. branch: remove_array_with_char_test
+.. branch: fix_test_unicode_outofrange
+.. branch: Ram-Rachum/faulthandleris_enabled-should-return-fal-1563636614875
+.. branch: Anthony-Sottile/fix-leak-of-file-descriptor-with-_iofile-1559687440863
+
+.. branch: py3tests
+
+Add handling of application-level test files and -D flag to test runner
+
+.. branch: vendor/stdlib-3.6
+.. branch: stdlib-3.6.9
+
+Update standard library to version 3.6.9
+
+.. branch: __debug__-optimize
+
+Fix handling of __debug__, sys.flags.optimize, and '-O' command-line flag to 
+match CPython 3.6.
+
+.. branch: more-cpyext
+
+Add ``PyErr_SetFromWindowsErr`` and ``pytime.h``, ``pytime.c``. Fix order of
+fields in ``Py_buffer``.
+
+.. branch: Ryan-Hileman/add-support-for-zipfile-stdlib-1562420744699
+
+Add support for the entire stdlib being inside a zipfile
+
+
+.. branch: json-decoder-maps-py3.6
+
+Much faster and more memory-efficient JSON decoding. The resulting
+dictionaries that come out of the JSON decoder have faster lookups too.
+
+

From pypy.commits at gmail.com  Fri Oct 18 13:03:48 2019
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 18 Oct 2019 10:03:48 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: Fix test
Message-ID: <5da9f074.1c69fb81.370bf.5202@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6
Changeset: r97812:acd72480cb0e
Date: 2019-10-18 18:02 +0100
http://bitbucket.org/pypy/pypy/changeset/acd72480cb0e/

Log:	Fix test

diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -510,7 +510,7 @@
         app_f = gateway.interp2app_temp(f, unwrap_spec=['fsencode'])
         w_app_f = space.wrap(app_f)
         if sys.platform == 'win32':
-            raises(gateway.OperationError, space.call_function, 
+            raises(gateway.OperationError, space.call_function,
                    w_app_f, w(u'\udc80'))
         else:
             assert space.eq_w(
@@ -1159,24 +1159,16 @@
         dict.update(d, **{clash: 33})
 
 
-
 class AppTestFastPathCrash(object):
-    def setup_class(cls):
-        cls.w_runappdirect = cls.space.wrap(cls.runappdirect)
-
     def test_fast_path_crash(self):
         # issue bb-3091 crash in BuiltinCodePassThroughArguments0.funcrun
+        import sys
+        if '__pypy__' in sys.modules:
+            msg_fmt = "'%s' object expected, got '%s'"
+        else:
+            msg_fmt = "'%s' object but received a '%s'"
         for obj in (dict, set):
             with raises(TypeError) as excinfo:
-                if self.runappdirect:
-                    import platform
-                    if platform.python_implementation() == 'PyPy':
-                        msg_fmt = "%s instance as first argument (got %s"
-                    else:
-                        msg_fmt = "'%s' object but received a '%s'"
-                    obj.__init__(0)
-                else:
-                    msg_fmt = "'%s' object expected, got '%s'"
-                    obj.__init__.im_func(0)
-            msg = msg_fmt %(obj.__name__, 'int')
+                obj.__init__(0)
+            msg = msg_fmt % (obj.__name__, 'int')
             assert msg in str(excinfo.value)

From pypy.commits at gmail.com  Fri Oct 18 15:15:51 2019
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 18 Oct 2019 12:15:51 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: fix inconsistent duplication in test:
 '\u00fc' and '\xFC' are the same thing
Message-ID: <5daa0f67.1c69fb81.5b8d8.3643@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6
Changeset: r97814:656f4b5c0802
Date: 2019-10-18 20:14 +0100
http://bitbucket.org/pypy/pypy/changeset/656f4b5c0802/

Log:	fix inconsistent duplication in test: '\u00fc' and '\xFC' are the
	same thing

diff --git a/pypy/module/_sre/test/test_app_sre.py b/pypy/module/_sre/test/test_app_sre.py
--- a/pypy/module/_sre/test/test_app_sre.py
+++ b/pypy/module/_sre/test/test_app_sre.py
@@ -809,7 +809,7 @@
             s.assert_no_match(opcodes, ["blaja", ""])
             opcodes = [s.OPCODES["at"], s.ATCODES[atname]] \
                 + s.encode_literal("bla") + [s.OPCODES["success"]]
-            s.assert_match(opcodes, "bla")
+            assert s.search(opcodes, "bla")
             s.assert_no_match(opcodes, "")
 
     def test_at_non_boundary(self):
@@ -817,7 +817,7 @@
         for atname in "at_non_boundary", "at_loc_non_boundary", "at_uni_non_boundary":
             opcodes = s.encode_literal("bla") \
                 + [s.OPCODES["at"], s.ATCODES[atname], s.OPCODES["success"]]
-            s.assert_match(opcodes, "blan")
+            assert s.search(opcodes, "blan")
             s.assert_no_match(opcodes, ["bla ja", "bla"])
 
     def test_at_loc_boundary(self):
@@ -829,12 +829,12 @@
                 + [s.OPCODES["at"], s.ATCODES["at_loc_boundary"], s.OPCODES["success"]]
             opcodes2 = s.encode_literal("bla") \
                 + [s.OPCODES["at"], s.ATCODES["at_loc_non_boundary"], s.OPCODES["success"]]
-            s.assert_match(opcodes1, "bla\xFC")
+            assert s.search(opcodes1, "bla\xFC")
             s.assert_no_match(opcodes2, "bla\xFC")
             oldlocale = locale.setlocale(locale.LC_ALL)
             locale.setlocale(locale.LC_ALL, "de_DE")
             s.assert_no_match(opcodes1, "bla\xFC")
-            s.assert_match(opcodes2, "bla\xFC")
+            assert s.search(opcodes2, "bla\xFC")
             locale.setlocale(locale.LC_ALL, oldlocale)
         except locale.Error:
             # skip test
@@ -861,13 +861,11 @@
                 + [s.OPCODES["category"], s.CHCODES["category_loc_word"], s.OPCODES["success"]]
             opcodes2 = s.encode_literal("b") \
                 + [s.OPCODES["category"], s.CHCODES["category_loc_not_word"], s.OPCODES["success"]]
-            s.assert_no_match(opcodes1, "b\xFC")
-            s.assert_no_match(opcodes1, "b\u00FC")
-            s.assert_match(opcodes2, "b\xFC")
+            assert not s.search(opcodes1, u"b\xFC")
+            assert s.search(opcodes2, u"b\xFC")
             locale.setlocale(locale.LC_ALL, "de_DE")
-            s.assert_match(opcodes1, "b\xFC")
-            s.assert_no_match(opcodes1, "b\u00FC")
-            s.assert_no_match(opcodes2, "b\xFC")
+            assert s.search(opcodes1, u"b\xFC")
+            assert not s.search(opcodes2, u"b\xFC")
             s.void_locale()
         except locale.Error:
             # skip test
@@ -1083,7 +1081,7 @@
     """These tests try to trigger optmized edge cases."""
 
     spaceconfig = {'usemodules': ['itertools']}
-    
+
     def test_match_length_optimization(self):
         import re
         assert None == re.match("bla", "blub")

From pypy.commits at gmail.com  Sat Oct 19 04:00:41 2019
From: pypy.commits at gmail.com (arigo)
Date: Sat, 19 Oct 2019 01:00:41 -0700 (PDT)
Subject: [pypy-commit] cffi default: Issue #427
Message-ID: <5daac2a9.1c69fb81.fc1ea.2159@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3299:7d1496495e25
Date: 2019-10-19 10:00 +0200
http://bitbucket.org/cffi/cffi/changeset/7d1496495e25/

Log:	Issue #427

	Fix: it's invalid according the API to call
	PyGILState_Release(PyGILState_UNLOCKED) manually at startup to
	release the GIL. Instead we must call another function, like
	PyEval_SaveThread(). The difference shows up in a rare case on
	CPython 3.7 only.

diff --git a/cffi/_embedding.h b/cffi/_embedding.h
--- a/cffi/_embedding.h
+++ b/cffi/_embedding.h
@@ -327,13 +327,15 @@
 #endif
 
     /* call Py_InitializeEx() */
-    {
-        PyGILState_STATE state = PyGILState_UNLOCKED;
-        if (!Py_IsInitialized())
-            _cffi_py_initialize();
-        else
-            state = PyGILState_Ensure();
-
+    if (!Py_IsInitialized()) {
+        _cffi_py_initialize();
+        PyEval_InitThreads();
+        PyEval_SaveThread();  /* release the GIL */
+        /* the returned tstate must be the one that has been stored into the
+           autoTLSkey by _PyGILState_Init() called from Py_Initialize(). */
+    }
+    else {
+        PyGILState_STATE state = PyGILState_Ensure();
         PyEval_InitThreads();
         PyGILState_Release(state);
     }

From pypy.commits at gmail.com  Sat Oct 19 10:56:51 2019
From: pypy.commits at gmail.com (JulianB)
Date: Sat, 19 Oct 2019 07:56:51 -0700 (PDT)
Subject: [pypy-commit] pypy license-update: Mention the newer directories as
 well under MIT in the license.
Message-ID: <5dab2433.1c69fb81.2c06c.b95c@mx.google.com>

Author: Julian Berman <Julian+Hg at GrayVines.com>
Branch: license-update
Changeset: r97816:dc6e2204f014
Date: 2019-10-19 10:55 -0400
http://bitbucket.org/pypy/pypy/changeset/dc6e2204f014/

Log:	Mention the newer directories as well under MIT in the license.

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -3,10 +3,11 @@
 License
 =======
 
-Except when otherwise stated (look for LICENSE files in directories or
-information at the beginning of each file) all software and documentation in
-the 'rpython', 'pypy', 'ctype_configure', 'dotviewer', 'demo', 'lib_pypy',
-'py', and '_pytest' directories is licensed as follows:
+Except when otherwise stated (look for LICENSE files in directories
+or information at the beginning of each file) all software and
+documentation in the 'rpython', 'pypy', 'ctype_configure', 'dotviewer',
+'demo', 'extra_tests', 'include', 'lib_pypy', 'py', and '_pytest'
+directories is licensed as follows:
 
     The MIT License
 

From pypy.commits at gmail.com  Sat Oct 19 11:02:57 2019
From: pypy.commits at gmail.com (JulianB)
Date: Sat, 19 Oct 2019 08:02:57 -0700 (PDT)
Subject: [pypy-commit] pypy https-readme: HTTP -> HTTPS the readme.
Message-ID: <5dab25a1.1c69fb81.eef08.2871@mx.google.com>

Author: Julian Berman <Julian+Hg at GrayVines.com>
Branch: https-readme
Changeset: r97817:35efaa242484
Date: 2019-10-19 11:02 -0400
http://bitbucket.org/pypy/pypy/changeset/35efaa242484/

Log:	HTTP -> HTTPS the readme.

diff --git a/README.rst b/README.rst
--- a/README.rst
+++ b/README.rst
@@ -9,15 +9,15 @@
 
 The home page for the interpreter is:
 
-    http://pypy.org/
+    https://pypy.org/
 
 If you want to help developing PyPy, this documentation might help you:
 
-    http://doc.pypy.org/
+    https://doc.pypy.org/
 
 More documentation about the RPython framework can be found here:
 
-    http://rpython.readthedocs.io/
+    https://rpython.readthedocs.io/
 
 The source for the documentation is in the pypy/doc directory.
 
@@ -25,7 +25,7 @@
 Using PyPy instead of CPython
 -----------------------------
 
-Please read the information at http://pypy.org/ to find the correct way to
+Please read the information at https://pypy.org/ to find the correct way to
 download and use PyPy as an alternative to CPython. 
 
 
@@ -36,7 +36,7 @@
 interpreter. It is time-consuming and requires significant computing resources.
 More information can be found here:
 
-    http://doc.pypy.org/en/latest/build.html
+    https://doc.pypy.org/en/latest/build.html
 
 Enjoy and send us feedback!
 

From pypy.commits at gmail.com  Sun Oct 20 02:39:46 2019
From: pypy.commits at gmail.com (arigo)
Date: Sat, 19 Oct 2019 23:39:46 -0700 (PDT)
Subject: [pypy-commit] pypy py3.7: Merged in
 Yannick_Jadoul/pypy/py3.7-bpo-29839 (pull request #674)
Message-ID: <5dac0132.1c69fb81.5387b.45ed@mx.google.com>

Author: Armin Rigo <armin.rigo at gmail.com>
Branch: py3.7
Changeset: r97822:357c87cee195
Date: 2019-10-20 06:39 +0000
http://bitbucket.org/pypy/pypy/changeset/357c87cee195/

Log:	Merged in Yannick_Jadoul/pypy/py3.7-bpo-29839 (pull request #674)

	bpo-29839 implementation: Avoid raising OverflowError in len() when
	__len__() returns negative large value

diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py
--- a/pypy/module/__builtin__/test/test_builtin.py
+++ b/pypy/module/__builtin__/test/test_builtin.py
@@ -751,6 +751,21 @@
             __dict__ = property(fget=getDict)
         assert vars(C_get_vars()) == {'a':2}
 
+    def test_len_negative_overflow(self):
+        import sys
+        class NegativeLen:
+            def __len__(self):
+                return -10
+        raises(ValueError, len, NegativeLen())
+        class HugeLen:
+            def __len__(self):
+                return sys.maxsize + 1
+        raises(OverflowError, len, HugeLen())
+        class HugeNegativeLen:
+            def __len__(self):
+                return -sys.maxsize-10
+        raises(ValueError, len, HugeNegativeLen())
+
 
 class AppTestGetattr:
     spaceconfig = {}
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -267,9 +267,12 @@
 
     def _check_len_result(space, w_obj):
         # Will complain if result is too big.
-        result = space.getindex_w(w_obj, space.w_OverflowError)
-        if result < 0:
+        w_result = space.index(w_obj)
+        assert space.isinstance_w(w_result, space.w_int)
+        if space.is_true(space.lt(w_result, space.newint(0))):
             raise oefmt(space.w_ValueError, "__len__() should return >= 0")
+        result = space.getindex_w(w_result, space.w_OverflowError)
+        assert result >= 0
         return result
 
     def is_iterable(space, w_obj):

From pypy.commits at gmail.com  Sun Oct 20 02:39:49 2019
From: pypy.commits at gmail.com (Yannick_Jadoul)
Date: Sat, 19 Oct 2019 23:39:49 -0700 (PDT)
Subject: [pypy-commit] pypy py3.7-bpo-29839: First check the result of
 __len__ for negative values rather before checking for overflow,
 implementing bpo-29839
Message-ID: <5dac0135.1c69fb81.44c2e.d075@mx.google.com>

Author: Yannick Jadoul <yannick.jadoul at belgacom.net>
Branch: py3.7-bpo-29839
Changeset: r97818:dd823d097737
Date: 2019-10-18 17:30 +0200
http://bitbucket.org/pypy/pypy/changeset/dd823d097737/

Log:	First check the result of __len__ for negative values rather before
	checking for overflow, implementing bpo-29839

diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -267,9 +267,12 @@
 
     def _check_len_result(space, w_obj):
         # Will complain if result is too big.
-        result = space.getindex_w(w_obj, space.w_OverflowError)
-        if result < 0:
+        w_result = space.index(w_obj)
+        assert space.isinstance_w(w_result, space.w_int)
+        if space.unwrap(w_result) < 0:
             raise oefmt(space.w_ValueError, "__len__() should return >= 0")
+        result = space.getindex_w(w_result, space.w_OverflowError)
+        assert result >= 0
         return result
 
     def is_iterable(space, w_obj):

From pypy.commits at gmail.com  Sun Oct 20 02:39:51 2019
From: pypy.commits at gmail.com (Yannick_Jadoul)
Date: Sat, 19 Oct 2019 23:39:51 -0700 (PDT)
Subject: [pypy-commit] pypy py3.7-bpo-29839: Added (well,
 copied) apptest for len
Message-ID: <5dac0137.1c69fb81.378f9.0955@mx.google.com>

Author: Yannick Jadoul <yannick.jadoul at belgacom.net>
Branch: py3.7-bpo-29839
Changeset: r97819:81794cb4ab7f
Date: 2019-10-18 19:30 +0200
http://bitbucket.org/pypy/pypy/changeset/81794cb4ab7f/

Log:	Added (well, copied) apptest for len

diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py
--- a/pypy/module/__builtin__/test/test_builtin.py
+++ b/pypy/module/__builtin__/test/test_builtin.py
@@ -751,6 +751,41 @@
             __dict__ = property(fget=getDict)
         assert vars(C_get_vars()) == {'a':2}
 
+    def test_len(self):
+        import sys
+        assert len('123') == 3
+        assert len(()) == 0
+        assert len((1, 2, 3, 4)) == 4
+        assert len([1, 2, 3, 4]) == 4
+        assert len({}) == 0
+        assert len({'a':1, 'b': 2}) == 2
+        class BadSeq:
+            def __len__(self):
+                raise ValueError
+        raises(ValueError, len, BadSeq())
+        class InvalidLen:
+            def __len__(self):
+                return None
+        raises(TypeError, len, InvalidLen())
+        class FloatLen:
+            def __len__(self):
+                return 4.5
+        raises(TypeError, len, FloatLen())
+        class NegativeLen:
+            def __len__(self):
+                return -10
+        raises(ValueError, len, NegativeLen())
+        class HugeLen:
+            def __len__(self):
+                return sys.maxsize + 1
+        raises(OverflowError, len, HugeLen())
+        class HugeNegativeLen:
+            def __len__(self):
+                return -sys.maxsize-10
+        raises(ValueError, len, HugeNegativeLen())
+        class NoLenMethod(object): pass
+        raises(TypeError, len, NoLenMethod())
+
 
 class AppTestGetattr:
     spaceconfig = {}

From pypy.commits at gmail.com  Sun Oct 20 02:39:52 2019
From: pypy.commits at gmail.com (Yannick_Jadoul)
Date: Sat, 19 Oct 2019 23:39:52 -0700 (PDT)
Subject: [pypy-commit] pypy py3.7-bpo-29839: Shortening
 test_builtin.py::AppTestBuiltinApp::test_len to more specific
 test_len_negative_overflow
Message-ID: <5dac0138.1c69fb81.c8cbf.3c7c@mx.google.com>

Author: Yannick Jadoul <yannick.jadoul at belgacom.net>
Branch: py3.7-bpo-29839
Changeset: r97820:f5ea45e7bf9c
Date: 2019-10-19 01:31 +0200
http://bitbucket.org/pypy/pypy/changeset/f5ea45e7bf9c/

Log:	Shortening test_builtin.py::AppTestBuiltinApp::test_len to more
	specific test_len_negative_overflow

diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py
--- a/pypy/module/__builtin__/test/test_builtin.py
+++ b/pypy/module/__builtin__/test/test_builtin.py
@@ -751,26 +751,8 @@
             __dict__ = property(fget=getDict)
         assert vars(C_get_vars()) == {'a':2}
 
-    def test_len(self):
+    def test_len_negative_overflow(self):
         import sys
-        assert len('123') == 3
-        assert len(()) == 0
-        assert len((1, 2, 3, 4)) == 4
-        assert len([1, 2, 3, 4]) == 4
-        assert len({}) == 0
-        assert len({'a':1, 'b': 2}) == 2
-        class BadSeq:
-            def __len__(self):
-                raise ValueError
-        raises(ValueError, len, BadSeq())
-        class InvalidLen:
-            def __len__(self):
-                return None
-        raises(TypeError, len, InvalidLen())
-        class FloatLen:
-            def __len__(self):
-                return 4.5
-        raises(TypeError, len, FloatLen())
         class NegativeLen:
             def __len__(self):
                 return -10
@@ -783,8 +765,6 @@
             def __len__(self):
                 return -sys.maxsize-10
         raises(ValueError, len, HugeNegativeLen())
-        class NoLenMethod(object): pass
-        raises(TypeError, len, NoLenMethod())
 
 
 class AppTestGetattr:

From pypy.commits at gmail.com  Sun Oct 20 02:39:54 2019
From: pypy.commits at gmail.com (Yannick_Jadoul)
Date: Sat, 19 Oct 2019 23:39:54 -0700 (PDT)
Subject: [pypy-commit] pypy py3.7-bpo-29839: Fix translation by avoiding
 space.unwrap
Message-ID: <5dac013a.1c69fb81.370bf.ea0e@mx.google.com>

Author: Yannick Jadoul <yannick.jadoul at belgacom.net>
Branch: py3.7-bpo-29839
Changeset: r97821:25e4f3f9050d
Date: 2019-10-19 20:29 +0200
http://bitbucket.org/pypy/pypy/changeset/25e4f3f9050d/

Log:	Fix translation by avoiding space.unwrap

diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -269,7 +269,7 @@
         # Will complain if result is too big.
         w_result = space.index(w_obj)
         assert space.isinstance_w(w_result, space.w_int)
-        if space.unwrap(w_result) < 0:
+        if space.is_true(space.lt(w_result, space.newint(0))):
             raise oefmt(space.w_ValueError, "__len__() should return >= 0")
         result = space.getindex_w(w_result, space.w_OverflowError)
         assert result >= 0

From pypy.commits at gmail.com  Sun Oct 20 07:49:54 2019
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 20 Oct 2019 04:49:54 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: Backport 3.7.5 changes to
 timedelta.__eq__ and time.__eq__ (bpo-37579)
Message-ID: <5dac49e2.1c69fb81.92dd0.0138@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6
Changeset: r97823:3f4502352ebe
Date: 2019-10-20 12:49 +0100
http://bitbucket.org/pypy/pypy/changeset/3f4502352ebe/

Log:	Backport 3.7.5 changes to timedelta.__eq__ and time.__eq__
	(bpo-37579)

diff --git a/lib-python/3/datetime.py b/lib-python/3/datetime.py
--- a/lib-python/3/datetime.py
+++ b/lib-python/3/datetime.py
@@ -609,7 +609,7 @@
         if isinstance(other, timedelta):
             return self._cmp(other) == 0
         else:
-            return False
+            return NotImplemented
 
     def __le__(self, other):
         if isinstance(other, timedelta):
@@ -1144,7 +1144,7 @@
         if isinstance(other, time):
             return self._cmp(other, allow_mixed=True) == 0
         else:
-            return False
+            return NotImplemented
 
     def __le__(self, other):
         if isinstance(other, time):

From pypy.commits at gmail.com  Sun Oct 20 08:08:53 2019
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 20 Oct 2019 05:08:53 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: Backport more fixes to comparisons in
 datetime.py (bpo-37985)
Message-ID: <5dac4e55.1c69fb81.60d08.397d@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6
Changeset: r97824:405ad10a4ddc
Date: 2019-10-20 13:08 +0100
http://bitbucket.org/pypy/pypy/changeset/405ad10a4ddc/

Log:	Backport more fixes to comparisons in datetime.py (bpo-37985)

diff --git a/lib-python/3/datetime.py b/lib-python/3/datetime.py
--- a/lib-python/3/datetime.py
+++ b/lib-python/3/datetime.py
@@ -615,25 +615,25 @@
         if isinstance(other, timedelta):
             return self._cmp(other) <= 0
         else:
-            _cmperror(self, other)
+            return NotImplemented
 
     def __lt__(self, other):
         if isinstance(other, timedelta):
             return self._cmp(other) < 0
         else:
-            _cmperror(self, other)
+            return NotImplemented
 
     def __ge__(self, other):
         if isinstance(other, timedelta):
             return self._cmp(other) >= 0
         else:
-            _cmperror(self, other)
+            return NotImplemented
 
     def __gt__(self, other):
         if isinstance(other, timedelta):
             return self._cmp(other) > 0
         else:
-            _cmperror(self, other)
+            return NotImplemented
 
     def _cmp(self, other):
         assert isinstance(other, timedelta)
@@ -1150,25 +1150,25 @@
         if isinstance(other, time):
             return self._cmp(other) <= 0
         else:
-            _cmperror(self, other)
+            return NotImplemented
 
     def __lt__(self, other):
         if isinstance(other, time):
             return self._cmp(other) < 0
         else:
-            _cmperror(self, other)
+            return NotImplemented
 
     def __ge__(self, other):
         if isinstance(other, time):
             return self._cmp(other) >= 0
         else:
-            _cmperror(self, other)
+            return NotImplemented
 
     def __gt__(self, other):
         if isinstance(other, time):
             return self._cmp(other) > 0
         else:
-            _cmperror(self, other)
+            return NotImplemented
 
     def _cmp(self, other, allow_mixed=False):
         assert isinstance(other, time)
@@ -2048,9 +2048,9 @@
         return (self._offset, self._name)
 
     def __eq__(self, other):
-        if type(other) != timezone:
-            return False
-        return self._offset == other._offset
+        if isinstance(other, timezone):
+            return self._offset == other._offset
+        return NotImplemented
 
     def __hash__(self):
         return hash(self._offset)

From pypy.commits at gmail.com  Mon Oct 21 05:24:24 2019
From: pypy.commits at gmail.com (arigo)
Date: Mon, 21 Oct 2019 02:24:24 -0700 (PDT)
Subject: [pypy-commit] cffi default: Issue #429
Message-ID: <5dad7948.1c69fb81.7d41f.9b33@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3301:3873bbdd3b9e
Date: 2019-10-21 11:24 +0200
http://bitbucket.org/cffi/cffi/changeset/3873bbdd3b9e/

Log:	Issue #429

	There are corner cases in which we can see a recursion on the same
	types. Instead of fighting them all, change the logic to complain if
	we recurse more than 1000 times.

diff --git a/c/realize_c_type.c b/c/realize_c_type.c
--- a/c/realize_c_type.c
+++ b/c/realize_c_type.c
@@ -631,14 +631,6 @@
         break;
     }
 
-    case 255:    /* recursion detection */
-        PyErr_Format(PyExc_RuntimeError,
-            "found a situation in which we try to build a type recursively.  "
-            "This is known to occur e.g. in ``struct s { void(*callable)"
-            "(struct s); }''.  Please report if you get this error and "
-            "really need support for your case.");
-        return NULL;
-
     default:
         PyErr_Format(PyExc_NotImplementedError, "op=%d", (int)_CFFI_GETOP(op));
         return NULL;
@@ -647,6 +639,8 @@
     return x;
 }
 
+static int _realize_recursion_level;
+
 static PyObject *
 realize_c_type_or_func(builder_c_t *builder,
                         _cffi_opcode_t opcodes[], int index)
@@ -660,10 +654,17 @@
         return x;
     }
 
-    opcodes[index] = (_cffi_opcode_t)255;   /* recursion detection */
+    if (_realize_recursion_level >= 1000) {
+        PyErr_Format(PyExc_RuntimeError,
+            "type-building recursion too deep or infinite.  "
+            "This is known to occur e.g. in ``struct s { void(*callable)"
+            "(struct s); }''.  Please report if you get this error and "
+            "really need support for your case.");
+        return NULL;
+    }
+    _realize_recursion_level++;
     x = realize_c_type_or_func_now(builder, op, opcodes, index);
-    if (opcodes[index] == (_cffi_opcode_t)255)
-        opcodes[index] = op;
+    _realize_recursion_level--;
 
     if (x != NULL && opcodes == builder->ctx.types && opcodes[index] != x) {
         assert((((uintptr_t)x) & 1) == 0);
diff --git a/testing/cffi1/test_re_python.py b/testing/cffi1/test_re_python.py
--- a/testing/cffi1/test_re_python.py
+++ b/testing/cffi1/test_re_python.py
@@ -75,6 +75,7 @@
     struct with_union { union { int a; char b; }; };
     union with_struct { struct { int a; char b; }; };
     struct NVGcolor { union { float rgba[4]; struct { float r,g,b,a; }; }; };
+    typedef struct selfref { struct selfref *next; } *selfref_ptr_t;
     """)
     ffi.set_source('re_python_pysrc', None)
     ffi.emit_python_code(str(tmpdir.join('re_python_pysrc.py')))
@@ -254,3 +255,8 @@
     assert ffi.offsetof("struct NVGcolor", "g") == FLOAT
     assert ffi.offsetof("struct NVGcolor", "b") == FLOAT * 2
     assert ffi.offsetof("struct NVGcolor", "a") == FLOAT * 3
+
+def test_selfref():
+    # based on issue #429
+    from re_python_pysrc import ffi
+    ffi.new("selfref_ptr_t")

From pypy.commits at gmail.com  Mon Oct 21 05:31:13 2019
From: pypy.commits at gmail.com (arigo)
Date: Mon, 21 Oct 2019 02:31:13 -0700 (PDT)
Subject: [pypy-commit] pypy default: manual update corresponding to
 cffi/3873bbdd3b9e
Message-ID: <5dad7ae1.1c69fb81.20a05.99fb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r97825:847f17dd11a6
Date: 2019-10-21 11:29 +0200
http://bitbucket.org/pypy/pypy/changeset/847f17dd11a6/

Log:	manual update corresponding to cffi/3873bbdd3b9e

diff --git a/pypy/module/_cffi_backend/realize_c_type.py b/pypy/module/_cffi_backend/realize_c_type.py
--- a/pypy/module/_cffi_backend/realize_c_type.py
+++ b/pypy/module/_cffi_backend/realize_c_type.py
@@ -83,6 +83,7 @@
         self.space = space
         self.all_primitives = [None] * cffi_opcode._NUM_PRIM
         self.file_struct = None
+        self.rec_level = 0
 
     def get_file_struct(self):
         if self.file_struct is None:
@@ -406,12 +407,18 @@
     if from_ffi and ffi.cached_types[index] is not None:
         return ffi.cached_types[index]
 
-    opcodes[index] = rffi.cast(rffi.VOIDP, 255)
+    realize_cache = ffi.space.fromcache(RealizeCache)
+    if realize_cache.rec_level >= 1000:
+        raise oefmt(ffi.space.w_RuntimeError,
+            "type-building recursion too deep or infinite.  "
+            "This is known to occur e.g. in ``struct s { void(*callable)"
+            "(struct s); }''.  Please report if you get this error and "
+            "really need support for your case.")
+    realize_cache.rec_level += 1
     try:
         x = realize_c_type_or_func_now(ffi, op, opcodes, index)
     finally:
-        if opcodes[index] == rffi.cast(rffi.VOIDP, 255):
-            opcodes[index] = op
+        realize_cache.rec_level -= 1
 
     if from_ffi:
         assert ffi.cached_types[index] is None or ffi.cached_types[index] is x
@@ -461,13 +468,6 @@
                                       'c_type_index')
         x = realize_c_type_or_func(ffi, ffi.ctxobj.ctx.c_types, type_index)
 
-    elif case == 255:
-        raise oefmt(ffi.space.w_RuntimeError,
-            "found a situation in which we try to build a type recursively.  "
-            "This is known to occur e.g. in ``struct s { void(*callable)"
-            "(struct s); }''.  Please report if you get this error and "
-            "really need support for your case.")
-
     else:
         raise oefmt(ffi.space.w_NotImplementedError, "op=%d", case)
 
diff --git a/pypy/module/_cffi_backend/test/test_re_python.py b/pypy/module/_cffi_backend/test/test_re_python.py
--- a/pypy/module/_cffi_backend/test/test_re_python.py
+++ b/pypy/module/_cffi_backend/test/test_re_python.py
@@ -74,6 +74,7 @@
         struct foo_s;
         typedef struct bar_s { int x; signed char a[]; } bar_t;
         enum foo_e { AA, BB, CC };
+        typedef struct selfref { struct selfref *next; } *selfref_ptr_t;
         """)
         ffi.set_source('re_python_pysrc', None)
         ffi.emit_python_code(str(tmpdir.join('re_python_pysrc.py')))
@@ -237,3 +238,9 @@
             "foobar", _version=0x2594)
         assert str(e.value).startswith(
             "cffi out-of-line Python module 'foobar' has unknown version")
+
+    def test_selfref(self):
+        # based on cffi issue #429
+        self.fix_path()
+        from re_python_pysrc import ffi
+        ffi.new("selfref_ptr_t")
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -2151,8 +2151,8 @@
             };
         """)
         e = raises(RuntimeError, ffi.new, "struct BinaryTree *")
-        assert str(e.value) == (
-            "found a situation in which we try to build a type recursively.  "
-            "This is known to occur e.g. in ``struct s { void(*callable)"
-            "(struct s); }''.  Please report if you get this error and "
-            "really need support for your case.")
+        # we should check e.value, but untranslated it crashes with a
+        # regular recursion error.  There is a chance it occurs translated
+        # too, but likely the check in the code ">= 1000" usually triggers
+        # before that, and raise a RuntimeError too, but with the more
+        # explicit message.

From pypy.commits at gmail.com  Mon Oct 21 05:31:15 2019
From: pypy.commits at gmail.com (arigo)
Date: Mon, 21 Oct 2019 02:31:15 -0700 (PDT)
Subject: [pypy-commit] pypy default: update all the way to cffi/3873bbdd3b9e
Message-ID: <5dad7ae3.1c69fb81.5ad6f.f114@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r97826:ec4f85880b7c
Date: 2019-10-21 11:30 +0200
http://bitbucket.org/pypy/pypy/changeset/ec4f85880b7c/

Log:	update all the way to cffi/3873bbdd3b9e

diff --git a/extra_tests/cffi_tests/cffi0/test_function.py b/extra_tests/cffi_tests/cffi0/test_function.py
--- a/extra_tests/cffi_tests/cffi0/test_function.py
+++ b/extra_tests/cffi_tests/cffi0/test_function.py
@@ -114,7 +114,7 @@
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
             int fputs(const char *, void *);
-            void *stderr;
+            extern void *stderr;
         """)
         needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
@@ -131,7 +131,7 @@
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
             int fputs(char *, void *);
-            void *stderr;
+            extern void *stderr;
         """)
         needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
@@ -148,7 +148,7 @@
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
            int fprintf(void *, const char *format, ...);
-           void *stderr;
+           extern void *stderr;
         """)
         needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
@@ -210,7 +210,7 @@
             py.test.skip("probably no symbol 'stderr' in the lib")
         ffi.cdef("""
             int fputs(const char *, void *);
-            void *stderr;
+            extern void *stderr;
         """)
         needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
@@ -257,7 +257,7 @@
             py.test.skip("probably no symbol 'stdout' in the lib")
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
-            void *stdout;
+            extern void *stdout;
         """)
         needs_dlopen_none()
         C = ffi.dlopen(None)
@@ -497,7 +497,7 @@
         ffi.cdef("""
             typedef enum { MYE1, MYE2 } myenum_t;
             double myfunc(double);
-            double myvar;
+            extern double myvar;
             const double myconst;
             #define MYFOO 42
         """)
@@ -508,7 +508,7 @@
         if self.Backend is CTypesBackend:
             py.test.skip("not with the ctypes backend")
         ffi = FFI(backend=self.Backend())
-        ffi.cdef("int foobar(void); int foobaz;")
+        ffi.cdef("int foobar(void); extern int foobaz;")
         lib = ffi.dlopen(lib_m)
         ffi.dlclose(lib)
         e = py.test.raises(ValueError, getattr, lib, 'foobar')
diff --git a/extra_tests/cffi_tests/cffi0/test_ownlib.py b/extra_tests/cffi_tests/cffi0/test_ownlib.py
--- a/extra_tests/cffi_tests/cffi0/test_ownlib.py
+++ b/extra_tests/cffi_tests/cffi0/test_ownlib.py
@@ -202,7 +202,7 @@
             py.test.skip("fix the auto-generation of the tiny test lib")
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
-            int my_array[7];
+            extern int my_array[7];
         """)
         ownlib = ffi.dlopen(self.module)
         for i in range(7):
@@ -224,7 +224,7 @@
             py.test.skip("not supported by the ctypes backend")
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
-            int my_array[];
+            extern int my_array[];
         """)
         ownlib = ffi.dlopen(self.module)
         for i in range(7):
@@ -292,7 +292,7 @@
                 long bottom;
             } RECT;
             
-            long left, top, right, bottom;
+            extern long left, top, right, bottom;
 
             RECT ReturnRect(int i, RECT ar, RECT* br, POINT cp, RECT dr,
                         RECT *er, POINT fp, RECT gr);
@@ -322,7 +322,7 @@
         if self.Backend is CTypesBackend:
             py.test.skip("not implemented with the ctypes backend")
         ffi = FFI(backend=self.Backend())
-        ffi.cdef("long left; int test_getting_errno(void);")
+        ffi.cdef("extern long left; int test_getting_errno(void);")
         lib = ffi.dlopen(self.module)
         lib.left = 123456
         p = ffi.addressof(lib, "left")
diff --git a/extra_tests/cffi_tests/cffi0/test_parsing.py b/extra_tests/cffi_tests/cffi0/test_parsing.py
--- a/extra_tests/cffi_tests/cffi0/test_parsing.py
+++ b/extra_tests/cffi_tests/cffi0/test_parsing.py
@@ -325,6 +325,7 @@
     assert value == sys.maxsize * 2 - 40
 
 def test__is_constant_globalvar():
+    import warnings
     for input, expected_output in [
         ("int a;",          False),
         ("const int a;",    True),
@@ -342,10 +343,13 @@
         ("const int a[5][6];", False),
         ]:
         ffi = FFI()
-        ffi.cdef(input)
+        with warnings.catch_warnings(record=True) as log:
+            warnings.simplefilter("always")
+            ffi.cdef(input)
         declarations = ffi._parser._declarations
         assert ('constant a' in declarations) == expected_output
         assert ('variable a' in declarations) == (not expected_output)
+        assert len(log) == (1 - expected_output)
 
 def test_restrict():
     from cffi import model
@@ -355,7 +359,7 @@
         ("int *a;",            False),
         ]:
         ffi = FFI()
-        ffi.cdef(input)
+        ffi.cdef("extern " + input)
         tp, quals = ffi._parser._declarations['variable a']
         assert bool(quals & model.Q_RESTRICT) == expected_output
 
diff --git a/extra_tests/cffi_tests/cffi0/test_verify.py b/extra_tests/cffi_tests/cffi0/test_verify.py
--- a/extra_tests/cffi_tests/cffi0/test_verify.py
+++ b/extra_tests/cffi_tests/cffi0/test_verify.py
@@ -287,7 +287,7 @@
 def test_var_signed_integer_types():
     ffi = FFI()
     lst = all_signed_integer_types(ffi)
-    csource = "\n".join(["%s somevar_%s;" % (tp, tp.replace(' ', '_'))
+    csource = "\n".join(["static %s somevar_%s;" % (tp, tp.replace(' ', '_'))
                          for tp in lst])
     ffi.cdef(csource)
     lib = ffi.verify(csource)
@@ -306,7 +306,7 @@
 def test_var_unsigned_integer_types():
     ffi = FFI()
     lst = all_unsigned_integer_types(ffi)
-    csource = "\n".join(["%s somevar_%s;" % (tp, tp.replace(' ', '_'))
+    csource = "\n".join(["static %s somevar_%s;" % (tp, tp.replace(' ', '_'))
                          for tp in lst])
     ffi.cdef(csource)
     lib = ffi.verify(csource)
@@ -818,8 +818,8 @@
 
 def test_access_variable():
     ffi = FFI()
-    ffi.cdef("int foo(void);\n"
-             "int somenumber;")
+    ffi.cdef("static int foo(void);\n"
+             "static int somenumber;")
     lib = ffi.verify("""
         static int somenumber = 2;
         static int foo(void) {
@@ -836,7 +836,7 @@
 def test_access_address_of_variable():
     # access the address of 'somenumber': need a trick
     ffi = FFI()
-    ffi.cdef("int somenumber; static int *const somenumberptr;")
+    ffi.cdef("static int somenumber; static int *const somenumberptr;")
     lib = ffi.verify("""
         static int somenumber = 2;
         #define somenumberptr (&somenumber)
@@ -849,7 +849,7 @@
 def test_access_array_variable(length=5):
     ffi = FFI()
     ffi.cdef("int foo(int);\n"
-             "int somenumber[%s];" % (length,))
+             "static int somenumber[%s];" % (length,))
     lib = ffi.verify("""
         static int somenumber[] = {2, 2, 3, 4, 5};
         static int foo(int i) {
@@ -881,7 +881,7 @@
     ffi = FFI()
     ffi.cdef("struct foo { int x; ...; };\n"
              "int foo(int);\n"
-             "struct foo stuff;")
+             "static struct foo stuff;")
     lib = ffi.verify("""
         struct foo { int x, y, z; };
         static struct foo stuff = {2, 5, 8};
@@ -905,9 +905,9 @@
 
 def test_access_callback():
     ffi = FFI()
-    ffi.cdef("int (*cb)(int);\n"
-             "int foo(int);\n"
-             "void reset_cb(void);")
+    ffi.cdef("static int (*cb)(int);\n"
+             "static int foo(int);\n"
+             "static void reset_cb(void);")
     lib = ffi.verify("""
         static int g(int x) { return x * 7; }
         static int (*cb)(int);
@@ -923,9 +923,9 @@
 def test_access_callback_function_typedef():
     ffi = FFI()
     ffi.cdef("typedef int mycallback_t(int);\n"
-             "mycallback_t *cb;\n"
-             "int foo(int);\n"
-             "void reset_cb(void);")
+             "static mycallback_t *cb;\n"
+             "static int foo(int);\n"
+             "static void reset_cb(void);")
     lib = ffi.verify("""
         static int g(int x) { return x * 7; }
         static int (*cb)(int);
@@ -1075,7 +1075,7 @@
 def test_autofilled_struct_as_argument_dynamic():
     ffi = FFI()
     ffi.cdef("struct foo_s { long a; ...; };\n"
-             "int (*foo)(struct foo_s);")
+             "static int (*foo)(struct foo_s);")
     lib = ffi.verify("""
         struct foo_s {
             double b;
@@ -1084,7 +1084,7 @@
         int foo1(struct foo_s s) {
             return (int)s.a - (int)s.b;
         }
-        int (*foo)(struct foo_s s) = &foo1;
+        static int (*foo)(struct foo_s s) = &foo1;
     """)
     e = py.test.raises(NotImplementedError, lib.foo, "?")
     msg = ("ctype 'struct foo_s' not supported as argument.  It is a struct "
@@ -1454,7 +1454,7 @@
         py.test.skip("_Bool not in MSVC")
     ffi = FFI()
     ffi.cdef("struct foo_s { _Bool x; };"
-             "_Bool foo(_Bool); _Bool (*foop)(_Bool);")
+             "_Bool foo(_Bool); static _Bool (*foop)(_Bool);")
     lib = ffi.verify("""
         struct foo_s { _Bool x; };
         int foo(int arg) {
@@ -1463,7 +1463,7 @@
         _Bool _foofunc(_Bool x) {
             return !x;
         }
-        _Bool (*foop)(_Bool) = _foofunc;
+        static _Bool (*foop)(_Bool) = _foofunc;
     """)
     p = ffi.new("struct foo_s *")
     p.x = 1
@@ -1654,7 +1654,7 @@
 
 def test_FILE_stored_explicitly():
     ffi = FFI()
-    ffi.cdef("int myprintf11(const char *, int); FILE *myfile;")
+    ffi.cdef("int myprintf11(const char *, int); extern FILE *myfile;")
     lib = ffi.verify("""
         #include <stdio.h>
         FILE *myfile;
@@ -1680,19 +1680,19 @@
 
 def test_global_array_with_missing_length():
     ffi = FFI()
-    ffi.cdef("int fooarray[];")
+    ffi.cdef("extern int fooarray[];")
     lib = ffi.verify("int fooarray[50];")
     assert repr(lib.fooarray).startswith("<cdata 'int *'")
 
 def test_global_array_with_dotdotdot_length():
     ffi = FFI()
-    ffi.cdef("int fooarray[...];")
+    ffi.cdef("extern int fooarray[...];")
     lib = ffi.verify("int fooarray[50];")
     assert repr(lib.fooarray).startswith("<cdata 'int[50]'")
 
 def test_bad_global_array_with_dotdotdot_length():
     ffi = FFI()
-    ffi.cdef("int fooarray[...];")
+    ffi.cdef("extern int fooarray[...];")
     py.test.raises(VerificationError, ffi.verify, "char fooarray[23];")
 
 def test_struct_containing_struct():
@@ -1813,7 +1813,7 @@
 def test_callback_indirection():
     ffi = FFI()
     ffi.cdef("""
-        int (*python_callback)(int how_many, int *values);
+        static int (*python_callback)(int how_many, int *values);
         int (*const c_callback)(int,...);   /* pass this ptr to C routines */
         int some_c_function(int(*cb)(int,...));
     """)
@@ -1947,24 +1947,24 @@
 
 def test_bug_const_char_ptr_array_1():
     ffi = FFI()
-    ffi.cdef("""const char *a[...];""")
+    ffi.cdef("""extern const char *a[...];""")
     lib = ffi.verify("""const char *a[5];""")
     assert repr(ffi.typeof(lib.a)) == "<ctype 'char *[5]'>"
 
 def test_bug_const_char_ptr_array_2():
     from cffi import FFI     # ignore warnings
     ffi = FFI()
-    ffi.cdef("""const int a[];""")
+    ffi.cdef("""extern const int a[];""")
     lib = ffi.verify("""const int a[5];""")
     assert repr(ffi.typeof(lib.a)) == "<ctype 'int *'>"
 
 def _test_various_calls(force_libffi):
     cdef_source = """
-    int xvalue;
-    long long ivalue, rvalue;
-    float fvalue;
-    double dvalue;
-    long double Dvalue;
+    extern int xvalue;
+    extern long long ivalue, rvalue;
+    extern float fvalue;
+    extern double dvalue;
+    extern long double Dvalue;
     signed char tf_bb(signed char x, signed char c);
     unsigned char tf_bB(signed char x, unsigned char c);
     short tf_bh(signed char x, short c);
@@ -2148,7 +2148,7 @@
     # exported symbols as well.  So we must not export a simple name
     # like 'foo'!
     ffi1 = FFI()
-    ffi1.cdef("int foo_verify_dlopen_flags;")
+    ffi1.cdef("extern int foo_verify_dlopen_flags;")
 
     lib1 = ffi1.verify("int foo_verify_dlopen_flags;",
                        flags=ffi1.RTLD_GLOBAL | ffi1.RTLD_LAZY)
@@ -2162,7 +2162,7 @@
 def get_second_lib():
     # Hack, using modulename makes the test fail
     ffi2 = FFI()
-    ffi2.cdef("int foo_verify_dlopen_flags;")
+    ffi2.cdef("extern int foo_verify_dlopen_flags;")
     lib2 = ffi2.verify("int foo_verify_dlopen_flags;",
                        flags=ffi2.RTLD_GLOBAL | ffi2.RTLD_LAZY)
     return lib2
diff --git a/extra_tests/cffi_tests/cffi1/test_dlopen.py b/extra_tests/cffi_tests/cffi1/test_dlopen.py
--- a/extra_tests/cffi_tests/cffi1/test_dlopen.py
+++ b/extra_tests/cffi_tests/cffi1/test_dlopen.py
@@ -7,7 +7,7 @@
 
 def test_simple():
     ffi = FFI()
-    ffi.cdef("int close(int); static const int BB = 42; int somevar;")
+    ffi.cdef("int close(int); static const int BB = 42; extern int somevar;")
     target = udir.join('test_simple.py')
     make_py_source(ffi, 'test_simple', str(target))
     assert target.read() == r"""# auto-generated file
@@ -197,7 +197,7 @@
 
 def test_global_var():
     ffi = FFI()
-    ffi.cdef("int myglob;")
+    ffi.cdef("extern int myglob;")
     target = udir.join('test_global_var.py')
     make_py_source(ffi, 'test_global_var', str(target))
     assert target.read() == r"""# auto-generated file
diff --git a/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py
--- a/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py
+++ b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py
@@ -1780,7 +1780,7 @@
 
     def test_import_from_lib(self):
         ffi2 = cffi.FFI()
-        ffi2.cdef("int myfunc(int); int myvar;\n#define MYFOO ...\n")
+        ffi2.cdef("int myfunc(int); extern int myvar;\n#define MYFOO ...\n")
         outputfilename = recompile(ffi2, "_test_import_from_lib",
                                    "int myfunc(int x) { return x + 1; }\n"
                                    "int myvar = -5;\n"
diff --git a/extra_tests/cffi_tests/cffi1/test_re_python.py b/extra_tests/cffi_tests/cffi1/test_re_python.py
--- a/extra_tests/cffi_tests/cffi1/test_re_python.py
+++ b/extra_tests/cffi_tests/cffi1/test_re_python.py
@@ -64,11 +64,11 @@
     #define BIGNEG -420000000000L
     int add42(int);
     int add43(int, ...);
-    int globalvar42;
+    extern int globalvar42;
     const int globalconst42;
     const char *const globalconsthello;
     int no_such_function(int);
-    int no_such_globalvar;
+    extern int no_such_globalvar;
     struct foo_s;
     typedef struct bar_s { int x; signed char a[]; } bar_t;
     enum foo_e { AA, BB, CC };
@@ -76,6 +76,7 @@
     struct with_union { union { int a; char b; }; };
     union with_struct { struct { int a; char b; }; };
     struct NVGcolor { union { float rgba[4]; struct { float r,g,b,a; }; }; };
+    typedef struct selfref { struct selfref *next; } *selfref_ptr_t;
     """)
     ffi.set_source('re_python_pysrc', None)
     ffi.emit_python_code(str(tmpdir.join('re_python_pysrc.py')))
@@ -255,3 +256,8 @@
     assert ffi.offsetof("struct NVGcolor", "g") == FLOAT
     assert ffi.offsetof("struct NVGcolor", "b") == FLOAT * 2
     assert ffi.offsetof("struct NVGcolor", "a") == FLOAT * 3
+
+def test_selfref():
+    # based on issue #429
+    from re_python_pysrc import ffi
+    ffi.new("selfref_ptr_t")
diff --git a/extra_tests/cffi_tests/cffi1/test_recompiler.py b/extra_tests/cffi_tests/cffi1/test_recompiler.py
--- a/extra_tests/cffi_tests/cffi1/test_recompiler.py
+++ b/extra_tests/cffi_tests/cffi1/test_recompiler.py
@@ -84,7 +84,7 @@
                      "(FUNCTION 1)(PRIMITIVE 7)(FUNCTION_END 1)(POINTER 0)")
 
 def test_type_table_array():
-    check_type_table("int a[100];",
+    check_type_table("extern int a[100];",
                      "(PRIMITIVE 7)(ARRAY 0)(None 100)")
 
 def test_type_table_typedef():
@@ -159,7 +159,7 @@
 
 def test_global_var_array():
     ffi = FFI()
-    ffi.cdef("int a[100];")
+    ffi.cdef("extern int a[100];")
     lib = verify(ffi, 'test_global_var_array', 'int a[100] = { 9999 };')
     lib.a[42] = 123456
     assert lib.a[42] == 123456
@@ -183,7 +183,7 @@
 
 def test_global_var_int():
     ffi = FFI()
-    ffi.cdef("int a, b, c;")
+    ffi.cdef("extern int a, b, c;")
     lib = verify(ffi, 'test_global_var_int', 'int a = 999, b, c;')
     assert lib.a == 999
     lib.a -= 1001
@@ -284,7 +284,7 @@
 
 def test_dir():
     ffi = FFI()
-    ffi.cdef("int ff(int); int aa; static const int my_constant;")
+    ffi.cdef("int ff(int); extern int aa; static const int my_constant;")
     lib = verify(ffi, 'test_dir', """
         #define my_constant  (-45)
         int aa;
@@ -406,7 +406,7 @@
 
 def test_dotdotdot_global_array():
     ffi = FFI()
-    ffi.cdef("int aa[...]; int bb[...];")
+    ffi.cdef("extern int aa[...]; extern int bb[...];")
     lib = verify(ffi, 'test_dotdotdot_global_array',
                  "int aa[41]; int bb[12];")
     assert ffi.sizeof(lib.aa) == 41 * 4
@@ -561,37 +561,37 @@
 
 def test_bad_size_of_global_1():
     ffi = FFI()
-    ffi.cdef("short glob;")
+    ffi.cdef("extern short glob;")
     py.test.raises(VerificationError, verify, ffi,
                    "test_bad_size_of_global_1", "long glob;")
 
 def test_bad_size_of_global_2():
     ffi = FFI()
-    ffi.cdef("int glob[10];")
+    ffi.cdef("extern int glob[10];")
     py.test.raises(VerificationError, verify, ffi,
                    "test_bad_size_of_global_2", "int glob[9];")
 
 def test_unspecified_size_of_global_1():
     ffi = FFI()
-    ffi.cdef("int glob[];")
+    ffi.cdef("extern int glob[];")
     lib = verify(ffi, "test_unspecified_size_of_global_1", "int glob[10];")
     assert ffi.typeof(lib.glob) == ffi.typeof("int *")
 
 def test_unspecified_size_of_global_2():
     ffi = FFI()
-    ffi.cdef("int glob[][5];")
+    ffi.cdef("extern int glob[][5];")
     lib = verify(ffi, "test_unspecified_size_of_global_2", "int glob[10][5];")
     assert ffi.typeof(lib.glob) == ffi.typeof("int(*)[5]")
 
 def test_unspecified_size_of_global_3():
     ffi = FFI()
-    ffi.cdef("int glob[][...];")
+    ffi.cdef("extern int glob[][...];")
     lib = verify(ffi, "test_unspecified_size_of_global_3", "int glob[10][5];")
     assert ffi.typeof(lib.glob) == ffi.typeof("int(*)[5]")
 
 def test_unspecified_size_of_global_4():
     ffi = FFI()
-    ffi.cdef("int glob[...][...];")
+    ffi.cdef("extern int glob[...][...];")
     lib = verify(ffi, "test_unspecified_size_of_global_4", "int glob[10][5];")
     assert ffi.typeof(lib.glob) == ffi.typeof("int[10][5]")
 
@@ -814,7 +814,7 @@
 def test_address_of_global_var():
     ffi = FFI()
     ffi.cdef("""
-        long bottom, bottoms[2];
+        extern long bottom, bottoms[2];
         long FetchRectBottom(void);
         long FetchRectBottoms1(void);
         #define FOOBAR 42
@@ -969,7 +969,7 @@
     ffi = FFI()
     ffi.cdef("""
         typedef ... opaque_t;
-        opaque_t globvar;
+        extern opaque_t globvar;
     """)
     lib = verify(ffi, 'test_variable_of_unknown_size', """
         typedef char opaque_t[6];
@@ -1014,7 +1014,7 @@
 def test_call_with_incomplete_structs():
     ffi = FFI()
     ffi.cdef("typedef struct {...;} foo_t; "
-             "foo_t myglob; "
+             "extern foo_t myglob; "
              "foo_t increment(foo_t s); "
              "double getx(foo_t s);")
     lib = verify(ffi, 'test_call_with_incomplete_structs', """
@@ -1058,7 +1058,7 @@
 
 def test_global_var_array_2():
     ffi = FFI()
-    ffi.cdef("int a[...][...];")
+    ffi.cdef("extern int a[...][...];")
     lib = verify(ffi, 'test_global_var_array_2', 'int a[10][8];')
     lib.a[9][7] = 123456
     assert lib.a[9][7] == 123456
@@ -1071,7 +1071,7 @@
 
 def test_global_var_array_3():
     ffi = FFI()
-    ffi.cdef("int a[][...];")
+    ffi.cdef("extern int a[][...];")
     lib = verify(ffi, 'test_global_var_array_3', 'int a[10][8];')
     lib.a[9][7] = 123456
     assert lib.a[9][7] == 123456
@@ -1082,7 +1082,7 @@
 
 def test_global_var_array_4():
     ffi = FFI()
-    ffi.cdef("int a[10][...];")
+    ffi.cdef("extern int a[10][...];")
     lib = verify(ffi, 'test_global_var_array_4', 'int a[10][8];')
     lib.a[9][7] = 123456
     assert lib.a[9][7] == 123456
@@ -1205,7 +1205,7 @@
 
 def test_import_from_lib():
     ffi = FFI()
-    ffi.cdef("int mybar(int); int myvar;\n#define MYFOO ...")
+    ffi.cdef("int mybar(int); static int myvar;\n#define MYFOO ...")
     lib = verify(ffi, 'test_import_from_lib',
                  "#define MYFOO 42\n"
                  "static int mybar(int x) { return x + 1; }\n"
@@ -1221,7 +1221,7 @@
 
 def test_macro_var_callback():
     ffi = FFI()
-    ffi.cdef("int my_value; int *(*get_my_value)(void);")
+    ffi.cdef("extern int my_value; extern int *(*get_my_value)(void);")
     lib = verify(ffi, 'test_macro_var_callback',
                  "int *(*get_my_value)(void);\n"
                  "#define my_value (*get_my_value())")
@@ -1336,7 +1336,7 @@
 
 def test_const_function_type_args():
     ffi = FFI()
-    ffi.cdef("""int (*foobar)(const int a, const int *b, const int c[]);""")
+    ffi.cdef("""extern int(*foobar)(const int a,const int*b,const int c[]);""")
     lib = verify(ffi, 'test_const_function_type_args', """
         int (*foobar)(const int a, const int *b, const int c[]);
     """)
@@ -1626,7 +1626,7 @@
 
 def test_extern_python_bogus_name():
     ffi = FFI()
-    ffi.cdef("int abc;")
+    ffi.cdef("extern int abc;")
     lib = verify(ffi, 'test_extern_python_bogus_name', "int abc;")
     def fn():
         pass
@@ -1787,8 +1787,8 @@
     ffi.cdef("""
         extern "Python" int __stdcall foo(int);
         extern "Python" int WINAPI bar(int);
-        int (__stdcall * mycb1)(int);
-        int indirect_call(int);
+        static int (__stdcall * mycb1)(int);
+        static int indirect_call(int);
     """)
     lib = verify(ffi, 'test_extern_python_stdcall', """
         #ifndef _MSC_VER
@@ -1856,7 +1856,7 @@
 
 def test_introspect_global_var():
     ffi = FFI()
-    ffi.cdef("float g1;")
+    ffi.cdef("extern float g1;")
     lib = verify(ffi, 'test_introspect_global_var', """
         float g1;
     """)
@@ -1867,7 +1867,7 @@
 
 def test_introspect_global_var_array():
     ffi = FFI()
-    ffi.cdef("float g1[100];")
+    ffi.cdef("extern float g1[100];")
     lib = verify(ffi, 'test_introspect_global_var_array', """
         float g1[100];
     """)
@@ -2090,7 +2090,7 @@
     ffi = FFI()
     ffi.cdef("""
         typedef int foo_t[...], bar_t[...];
-        int gv[...];
+        extern int gv[...];
         typedef int mat_t[...][...];
         typedef int vmat_t[][...];
         """)
diff --git a/extra_tests/cffi_tests/cffi1/test_verify1.py b/extra_tests/cffi_tests/cffi1/test_verify1.py
--- a/extra_tests/cffi_tests/cffi1/test_verify1.py
+++ b/extra_tests/cffi_tests/cffi1/test_verify1.py
@@ -268,7 +268,7 @@
 def test_var_signed_integer_types():
     ffi = FFI()
     lst = all_signed_integer_types(ffi)
-    csource = "\n".join(["%s somevar_%s;" % (tp, tp.replace(' ', '_'))
+    csource = "\n".join(["static %s somevar_%s;" % (tp, tp.replace(' ', '_'))
                          for tp in lst])
     ffi.cdef(csource)
     lib = ffi.verify(csource)
@@ -287,7 +287,7 @@
 def test_var_unsigned_integer_types():
     ffi = FFI()
     lst = all_unsigned_integer_types(ffi)
-    csource = "\n".join(["%s somevar_%s;" % (tp, tp.replace(' ', '_'))
+    csource = "\n".join(["static %s somevar_%s;" % (tp, tp.replace(' ', '_'))
                          for tp in lst])
     ffi.cdef(csource)
     lib = ffi.verify(csource)
@@ -791,8 +791,8 @@
 
 def test_access_variable():
     ffi = FFI()
-    ffi.cdef("int foo(void);\n"
-             "int somenumber;")
+    ffi.cdef("static int foo(void);\n"
+             "static int somenumber;")
     lib = ffi.verify("""
         static int somenumber = 2;
         static int foo(void) {
@@ -809,7 +809,7 @@
 def test_access_address_of_variable():
     # access the address of 'somenumber': need a trick
     ffi = FFI()
-    ffi.cdef("int somenumber; static int *const somenumberptr;")
+    ffi.cdef("static int somenumber; static int *const somenumberptr;")
     lib = ffi.verify("""
         static int somenumber = 2;
         #define somenumberptr (&somenumber)
@@ -821,8 +821,8 @@
 
 def test_access_array_variable(length=5):
     ffi = FFI()
-    ffi.cdef("int foo(int);\n"
-             "int somenumber[%s];" % (length,))
+    ffi.cdef("static int foo(int);\n"
+             "static int somenumber[%s];" % (length,))
     lib = ffi.verify("""
         static int somenumber[] = {2, 2, 3, 4, 5};
         static int foo(int i) {
@@ -853,8 +853,8 @@
 def test_access_struct_variable():
     ffi = FFI()
     ffi.cdef("struct foo { int x; ...; };\n"
-             "int foo(int);\n"
-             "struct foo stuff;")
+             "static int foo(int);\n"
+             "static struct foo stuff;")
     lib = ffi.verify("""
         struct foo { int x, y, z; };
         static struct foo stuff = {2, 5, 8};
@@ -878,9 +878,9 @@
 
 def test_access_callback():
     ffi = FFI()
-    ffi.cdef("int (*cb)(int);\n"
-             "int foo(int);\n"
-             "void reset_cb(void);")
+    ffi.cdef("static int (*cb)(int);\n"
+             "static int foo(int);\n"
+             "static void reset_cb(void);")
     lib = ffi.verify("""
         static int g(int x) { return x * 7; }
         static int (*cb)(int);
@@ -896,9 +896,9 @@
 def test_access_callback_function_typedef():
     ffi = FFI()
     ffi.cdef("typedef int mycallback_t(int);\n"
-             "mycallback_t *cb;\n"
-             "int foo(int);\n"
-             "void reset_cb(void);")
+             "static mycallback_t *cb;\n"
+             "static int foo(int);\n"
+             "static void reset_cb(void);")
     lib = ffi.verify("""
         static int g(int x) { return x * 7; }
         static int (*cb)(int);
@@ -1039,7 +1039,7 @@
 def test_autofilled_struct_as_argument_dynamic():
     ffi = FFI()
     ffi.cdef("struct foo_s { long a; ...; };\n"
-             "int (*foo)(struct foo_s);")
+             "static int (*foo)(struct foo_s);")
     lib = ffi.verify("""
         struct foo_s {
             double b;
@@ -1048,7 +1048,7 @@
         int foo1(struct foo_s s) {
             return (int)s.a - (int)s.b;
         }
-        int (*foo)(struct foo_s s) = &foo1;
+        static int (*foo)(struct foo_s s) = &foo1;
     """)
     e = py.test.raises(NotImplementedError, lib.foo, "?")
     msg = ("ctype 'struct foo_s' not supported as argument.  It is a struct "
@@ -1424,7 +1424,7 @@
         py.test.skip("_Bool not in MSVC")
     ffi = FFI()
     ffi.cdef("struct foo_s { _Bool x; };"
-             "_Bool foo(_Bool); _Bool (*foop)(_Bool);")
+             "_Bool foo(_Bool); static _Bool (*foop)(_Bool);")
     lib = ffi.verify("""
         struct foo_s { _Bool x; };
         int foo(int arg) {
@@ -1433,7 +1433,7 @@
         _Bool _foofunc(_Bool x) {
             return !x;
         }
-        _Bool (*foop)(_Bool) = _foofunc;
+        static _Bool (*foop)(_Bool) = _foofunc;
     """)
     p = ffi.new("struct foo_s *")
     p.x = 1
@@ -1618,7 +1618,7 @@
 
 def test_FILE_stored_explicitly():
     ffi = FFI()
-    ffi.cdef("int myprintf11(const char *, int); FILE *myfile;")
+    ffi.cdef("int myprintf11(const char *, int); extern FILE *myfile;")
     lib = ffi.verify("""
         #include <stdio.h>
         FILE *myfile;
@@ -1644,13 +1644,13 @@
 
 def test_global_array_with_missing_length():
     ffi = FFI()
-    ffi.cdef("int fooarray[];")
+    ffi.cdef("extern int fooarray[];")
     lib = ffi.verify("int fooarray[50];")
     assert repr(lib.fooarray).startswith("<cdata 'int *'")
 
 def test_global_array_with_dotdotdot_length():
     ffi = FFI()
-    ffi.cdef("int fooarray[...];")
+    ffi.cdef("extern int fooarray[...];")
     lib = ffi.verify("int fooarray[50];")
     assert repr(lib.fooarray).startswith("<cdata 'int[50]'")
 
@@ -1658,7 +1658,7 @@
     py.test.xfail("was detected only because 23 bytes cannot be divided by 4; "
                   "redo more generally")
     ffi = FFI()
-    ffi.cdef("int fooarray[...];")
+    ffi.cdef("extern int fooarray[...];")
     py.test.raises(VerificationError, ffi.verify, "char fooarray[23];")
 
 def test_struct_containing_struct():
@@ -1779,7 +1779,7 @@
 def test_callback_indirection():
     ffi = FFI()
     ffi.cdef("""
-        int (*python_callback)(int how_many, int *values);
+        static int (*python_callback)(int how_many, int *values);
         int (*const c_callback)(int,...);   /* pass this ptr to C routines */
         int some_c_function(int(*cb)(int,...));
     """)
@@ -1913,23 +1913,23 @@
 
 def test_bug_const_char_ptr_array_1():
     ffi = FFI()
-    ffi.cdef("""const char *a[...];""")
+    ffi.cdef("""extern const char *a[...];""")
     lib = ffi.verify("""const char *a[5];""")
     assert repr(ffi.typeof(lib.a)) == "<ctype 'char *[5]'>"
 
 def test_bug_const_char_ptr_array_2():
     ffi = FFI()
-    ffi.cdef("""const int a[];""")
+    ffi.cdef("""extern const int a[];""")
     lib = ffi.verify("""const int a[5];""")
     assert repr(ffi.typeof(lib.a)) == "<ctype 'int *'>"
 
 def _test_various_calls(force_libffi):
     cdef_source = """
-    int xvalue;
-    long long ivalue, rvalue;
-    float fvalue;
-    double dvalue;
-    long double Dvalue;
+    extern int xvalue;
+    extern long long ivalue, rvalue;
+    extern float fvalue;
+    extern double dvalue;
+    extern long double Dvalue;
     signed char tf_bb(signed char x, signed char c);
     unsigned char tf_bB(signed char x, unsigned char c);
     short tf_bh(signed char x, short c);
@@ -2112,7 +2112,7 @@
     old = sys.getdlopenflags()
     try:
         ffi1 = FFI()
-        ffi1.cdef("int foo_verify_dlopen_flags_1;")
+        ffi1.cdef("extern int foo_verify_dlopen_flags_1;")
         sys.setdlopenflags(ffi1.RTLD_GLOBAL | ffi1.RTLD_NOW)
         lib1 = ffi1.verify("int foo_verify_dlopen_flags_1;")
     finally:
@@ -2253,7 +2253,7 @@
 
 def test_macro_var():
     ffi = FFI()
-    ffi.cdef("int myarray[50], my_value;")
+    ffi.cdef("extern int myarray[50], my_value;")
     lib = ffi.verify("""
         int myarray[50];
         int *get_my_value(void) {
diff --git a/extra_tests/cffi_tests/embedding/add1.py b/extra_tests/cffi_tests/embedding/add1.py
--- a/extra_tests/cffi_tests/embedding/add1.py
+++ b/extra_tests/cffi_tests/embedding/add1.py
@@ -12,7 +12,11 @@
     sys.stdout.write("preparing")
     for i in range(3):
         sys.stdout.flush()
-        time.sleep(0.2)
+        # Windows: sometimes time.sleep() doesn't sleep at all.
+        # This appears to occur on recent versions of python only.
+        t_end = time.time() + 0.19
+        while time.time() < t_end:
+            time.sleep(0.2)
         sys.stdout.write(".")
     sys.stdout.write("\n")
 
diff --git a/extra_tests/cffi_tests/embedding/add_recursive.py b/extra_tests/cffi_tests/embedding/add_recursive.py
--- a/extra_tests/cffi_tests/embedding/add_recursive.py
+++ b/extra_tests/cffi_tests/embedding/add_recursive.py
@@ -4,7 +4,7 @@
 ffi = cffi.FFI()
 
 ffi.embedding_api("""
-    int (*my_callback)(int);
+    extern int (*my_callback)(int);
     int add_rec(int, int);
 """)
 
diff --git a/extra_tests/cffi_tests/embedding/test_thread.py b/extra_tests/cffi_tests/embedding/test_thread.py
--- a/extra_tests/cffi_tests/embedding/test_thread.py
+++ b/extra_tests/cffi_tests/embedding/test_thread.py
@@ -22,17 +22,21 @@
         add1_cffi = self.prepare_module('add1')
         add2_cffi = self.prepare_module('add2')
         self.compile('thread2-test', [add1_cffi, add2_cffi], threads=True)
-        output = self.execute('thread2-test')
-        output = self._take_out(output, "preparing")
-        output = self._take_out(output, ".")
-        output = self._take_out(output, ".")
-        # at least the 3rd dot should be after everything from ADD2
-        assert output == ("starting\n"
-                          "prepADD2\n"
-                          "adding 1000 and 200 and 30\n"
-                          ".\n"
-                          "adding 40 and 2\n"
-                          "done\n")
+        for i in range(3):
+            output = self.execute('thread2-test')
+            print('='*79)
+            print(output)
+            print('='*79)
+            output = self._take_out(output, "preparing")
+            output = self._take_out(output, ".")
+            output = self._take_out(output, ".")
+            # at least the 3rd dot should be after everything from ADD2
+            assert output == ("starting\n"
+                              "prepADD2\n"
+                              "adding 1000 and 200 and 30\n"
+                              ".\n"
+                              "adding 40 and 2\n"
+                              "done\n")
 
     def test_alt_issue(self):
         add1_cffi = self.prepare_module('add1')
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -327,13 +327,15 @@
 #endif
 
     /* call Py_InitializeEx() */
-    {
-        PyGILState_STATE state = PyGILState_UNLOCKED;
-        if (!Py_IsInitialized())
-            _cffi_py_initialize();
-        else
-            state = PyGILState_Ensure();
-
+    if (!Py_IsInitialized()) {
+        _cffi_py_initialize();
+        PyEval_InitThreads();
+        PyEval_SaveThread();  /* release the GIL */
+        /* the returned tstate must be the one that has been stored into the
+           autoTLSkey by _PyGILState_Init() called from Py_Initialize(). */
+    }
+    else {
+        PyGILState_STATE state = PyGILState_Ensure();
         PyEval_InitThreads();
         PyGILState_Release(state);
     }
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -156,6 +156,13 @@
                           "confuse pre-parsing.")
             break
 
+def _warn_for_non_extern_non_static_global_variable(decl):
+    if not decl.storage:
+        import warnings
+        warnings.warn("Declaration of global variable '%s' in cdef() should "
+                      "be marked 'extern' for consistency (or possibly "
+                      "'static' in API mode)" % (decl.name,))
+
 def _preprocess(csource):
     # Remove comments.  NOTE: this only work because the cdef() section
     # should not contain any string literal!
@@ -506,6 +513,7 @@
                     if (quals & model.Q_CONST) and not tp.is_array_type:
                         self._declare('constant ' + decl.name, tp, quals=quals)
                     else:
+                        _warn_for_non_extern_non_static_global_variable(decl)
                         self._declare('variable ' + decl.name, tp, quals=quals)
 
     def parse_type(self, cdecl):

From pypy.commits at gmail.com  Mon Oct 21 05:35:21 2019
From: pypy.commits at gmail.com (arigo)
Date: Mon, 21 Oct 2019 02:35:21 -0700 (PDT)
Subject: [pypy-commit] cffi default: update whatsnew
Message-ID: <5dad7bd9.1c69fb81.63380.b414@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3302:35d0ff266cf5
Date: 2019-10-21 11:35 +0200
http://bitbucket.org/cffi/cffi/changeset/35d0ff266cf5/

Log:	update whatsnew

diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -12,9 +12,13 @@
   the purposes of ``cdef()``, but the reason for deprecating the bare version
   is that (as far as I know) it would always be mistake in a real C header.
 
+* fix the regression ``RuntimeError: found a situation in which we try
+  to build a type recursively`` (`issue #429`_).
+
 * fixed `issue #427`_ where a multithreading mistake in the embedding logic
   initialization code would cause deadlocks on CPython 3.7.
 
+.. _`issue #429`: https://bitbucket.org/cffi/cffi/issues/429/
 .. _`issue #427`: https://bitbucket.org/cffi/cffi/issues/427/
 
 

From pypy.commits at gmail.com  Mon Oct 21 06:42:17 2019
From: pypy.commits at gmail.com (arigo)
Date: Mon, 21 Oct 2019 03:42:17 -0700 (PDT)
Subject: [pypy-commit] cffi default: bump version to 1.13.1
Message-ID: <5dad8b89.1c69fb81.5b009.4a9a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3303:dd09b2809748
Date: 2019-10-21 12:41 +0200
http://bitbucket.org/cffi/cffi/changeset/dd09b2809748/

Log:	bump version to 1.13.1

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -2,7 +2,7 @@
 #include <Python.h>
 #include "structmember.h"
 
-#define CFFI_VERSION  "1.13.0"
+#define CFFI_VERSION  "1.13.1"
 
 #ifdef MS_WIN32
 #include <windows.h>
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -14,7 +14,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.13.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.13.1", ("This test_c.py file is for testing a version"
                                  " of cffi that differs from the one that we"
                                  " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
diff --git a/cffi/__init__.py b/cffi/__init__.py
--- a/cffi/__init__.py
+++ b/cffi/__init__.py
@@ -5,8 +5,8 @@
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 from .error import PkgConfigError
 
-__version__ = "1.13.0"
-__version_info__ = (1, 13, 0)
+__version__ = "1.13.1"
+__version_info__ = (1, 13, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/cffi/_embedding.h b/cffi/_embedding.h
--- a/cffi/_embedding.h
+++ b/cffi/_embedding.h
@@ -224,7 +224,7 @@
 
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.13.0"
+                               "\ncompiled with cffi version: 1.13.1"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/doc/source/conf.py b/doc/source/conf.py
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -47,7 +47,7 @@
 # The short X.Y version.
 version = '1.13'
 # The full version, including alpha/beta/rc tags.
-release = '1.13.0'
+release = '1.13.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -52,13 +52,13 @@
 
 * https://pypi.python.org/pypi/cffi
 
-* Checksums of the "source" package version 1.13.0:
+* Checksums of the "source" package version 1.13.1:
 
-   - MD5: 2c90be6d3bfbf80ab732b28fd677051d
+   - MD5: ...
 
-   - SHA: 75977bbad8ea8c15f2c34edde5efcf39462ea43e
+   - SHA: ...
 
-   - SHA256: 8fe230f612c18af1df6f348d02d682fe2c28ca0a6c3856c99599cdacae7cf226
+   - SHA256: ...
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -198,7 +198,7 @@
 
 `Mailing list <https://groups.google.com/forum/#!forum/python-cffi>`_
 """,
-        version='1.13.0',
+        version='1.13.1',
         packages=['cffi'] if cpython else [],
         package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h', 
                                '_embedding.h', '_cffi_errors.h']}

From pypy.commits at gmail.com  Mon Oct 21 06:44:34 2019
From: pypy.commits at gmail.com (arigo)
Date: Mon, 21 Oct 2019 03:44:34 -0700 (PDT)
Subject: [pypy-commit] pypy default: bump cffi version to 1.13.1
Message-ID: <5dad8c12.1c69fb81.c0650.0b2d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r97827:b514063ad22a
Date: 2019-10-21 12:43 +0200
http://bitbucket.org/pypy/pypy/changeset/b514063ad22a/

Log:	bump cffi version to 1.13.1

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.13.0
+Version: 1.13.1
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -5,8 +5,8 @@
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 from .error import PkgConfigError
 
-__version__ = "1.13.0"
-__version_info__ = (1, 13, 0)
+__version__ = "1.13.1"
+__version_info__ = (1, 13, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -224,7 +224,7 @@
 
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.13.0"
+                               "\ncompiled with cffi version: 1.13.1"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -1,1 +1,1 @@
-VERSION = "1.13.0"
+VERSION = "1.13.1"
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.13.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.13.1", ("This test_c.py file is for testing a version"
                                  " of cffi that differs from the one that we"
                                  " get from 'import _cffi_backend'")
 if sys.version_info < (3,):

From pypy.commits at gmail.com  Mon Oct 21 10:57:52 2019
From: pypy.commits at gmail.com (arigo)
Date: Mon, 21 Oct 2019 07:57:52 -0700 (PDT)
Subject: [pypy-commit] cffi release-1.13: hg merge default
Message-ID: <5dadc770.1c69fb81.629b6.ea8f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-1.13
Changeset: r3304:bb9c3461e5a4
Date: 2019-10-21 12:44 +0200
http://bitbucket.org/cffi/cffi/changeset/bb9c3461e5a4/

Log:	hg merge default

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -2,7 +2,7 @@
 #include <Python.h>
 #include "structmember.h"
 
-#define CFFI_VERSION  "1.13.0"
+#define CFFI_VERSION  "1.13.1"
 
 #ifdef MS_WIN32
 #include <windows.h>
diff --git a/c/realize_c_type.c b/c/realize_c_type.c
--- a/c/realize_c_type.c
+++ b/c/realize_c_type.c
@@ -631,14 +631,6 @@
         break;
     }
 
-    case 255:    /* recursion detection */
-        PyErr_Format(PyExc_RuntimeError,
-            "found a situation in which we try to build a type recursively.  "
-            "This is known to occur e.g. in ``struct s { void(*callable)"
-            "(struct s); }''.  Please report if you get this error and "
-            "really need support for your case.");
-        return NULL;
-
     default:
         PyErr_Format(PyExc_NotImplementedError, "op=%d", (int)_CFFI_GETOP(op));
         return NULL;
@@ -647,6 +639,8 @@
     return x;
 }
 
+static int _realize_recursion_level;
+
 static PyObject *
 realize_c_type_or_func(builder_c_t *builder,
                         _cffi_opcode_t opcodes[], int index)
@@ -660,10 +654,17 @@
         return x;
     }
 
-    opcodes[index] = (_cffi_opcode_t)255;   /* recursion detection */
+    if (_realize_recursion_level >= 1000) {
+        PyErr_Format(PyExc_RuntimeError,
+            "type-building recursion too deep or infinite.  "
+            "This is known to occur e.g. in ``struct s { void(*callable)"
+            "(struct s); }''.  Please report if you get this error and "
+            "really need support for your case.");
+        return NULL;
+    }
+    _realize_recursion_level++;
     x = realize_c_type_or_func_now(builder, op, opcodes, index);
-    if (opcodes[index] == (_cffi_opcode_t)255)
-        opcodes[index] = op;
+    _realize_recursion_level--;
 
     if (x != NULL && opcodes == builder->ctx.types && opcodes[index] != x) {
         assert((((uintptr_t)x) & 1) == 0);
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -14,7 +14,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.13.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.13.1", ("This test_c.py file is for testing a version"
                                  " of cffi that differs from the one that we"
                                  " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
diff --git a/cffi/__init__.py b/cffi/__init__.py
--- a/cffi/__init__.py
+++ b/cffi/__init__.py
@@ -5,8 +5,8 @@
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 from .error import PkgConfigError
 
-__version__ = "1.13.0"
-__version_info__ = (1, 13, 0)
+__version__ = "1.13.1"
+__version_info__ = (1, 13, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/cffi/_embedding.h b/cffi/_embedding.h
--- a/cffi/_embedding.h
+++ b/cffi/_embedding.h
@@ -224,7 +224,7 @@
 
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.13.0"
+                               "\ncompiled with cffi version: 1.13.1"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
@@ -327,13 +327,15 @@
 #endif
 
     /* call Py_InitializeEx() */
-    {
-        PyGILState_STATE state = PyGILState_UNLOCKED;
-        if (!Py_IsInitialized())
-            _cffi_py_initialize();
-        else
-            state = PyGILState_Ensure();
-
+    if (!Py_IsInitialized()) {
+        _cffi_py_initialize();
+        PyEval_InitThreads();
+        PyEval_SaveThread();  /* release the GIL */
+        /* the returned tstate must be the one that has been stored into the
+           autoTLSkey by _PyGILState_Init() called from Py_Initialize(). */
+    }
+    else {
+        PyGILState_STATE state = PyGILState_Ensure();
         PyEval_InitThreads();
         PyGILState_Release(state);
     }
diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -156,6 +156,13 @@
                           "confuse pre-parsing.")
             break
 
+def _warn_for_non_extern_non_static_global_variable(decl):
+    if not decl.storage:
+        import warnings
+        warnings.warn("Declaration of global variable '%s' in cdef() should "
+                      "be marked 'extern' for consistency (or possibly "
+                      "'static' in API mode)" % (decl.name,))
+
 def _preprocess(csource):
     # Remove comments.  NOTE: this only work because the cdef() section
     # should not contain any string literal!
@@ -506,6 +513,7 @@
                     if (quals & model.Q_CONST) and not tp.is_array_type:
                         self._declare('constant ' + decl.name, tp, quals=quals)
                     else:
+                        _warn_for_non_extern_non_static_global_variable(decl)
                         self._declare('variable ' + decl.name, tp, quals=quals)
 
     def parse_type(self, cdecl):
diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst
--- a/doc/source/cdef.rst
+++ b/doc/source/cdef.rst
@@ -512,14 +512,14 @@
    field; then you would use "``typedef struct { ...; } foo_t;``".
 
 *  array lengths: when used as structure fields or in global variables,
-   arrays can have an unspecified length, as in "``int n[...];``".  The
+   arrays can have an unspecified length, as in "``extern int n[...];``".  The
    length is completed by the C compiler.
-   This is slightly different from "``int n[];``", because the latter
+   This is slightly different from "``extern int n[];``", because the latter
    means that the length is not known even to the C compiler, and thus
    no attempt is made to complete it.  This supports
-   multidimensional arrays: "``int n[...][...];``".
+   multidimensional arrays: "``extern int n[...][...];``".
 
-   *New in version 1.2:* "``int m[][...];``", i.e. ``...`` can be used
+   *New in version 1.2:* "``extern int m[][...];``", i.e. ``...`` can be used
    in the innermost dimensions without being also used in the outermost
    dimension.  In the example given, the length of the ``m`` array is
    assumed not to be known to the C compiler, but the length of every
@@ -568,8 +568,8 @@
 
 For more complex types, you have no choice but be precise.  For example,
 you cannot misdeclare a ``int *`` argument as ``long *``, or a global
-array ``int a[5];`` as ``long a[5];``.  CFFI considers `all types listed
-above`_ as primitive (so ``long long a[5];`` and ``int64_t a[5]`` are
+array ``extern int a[5];`` as ``extern long a[5];``.  CFFI considers `all types listed
+above`_ as primitive (so ``extern long long a[5];`` and ``extern int64_t a[5]`` are
 different declarations).  The reason for that is detailed in `a comment
 about an issue.`__
 
diff --git a/doc/source/conf.py b/doc/source/conf.py
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -47,7 +47,7 @@
 # The short X.Y version.
 version = '1.13'
 # The full version, including alpha/beta/rc tags.
-release = '1.13.0'
+release = '1.13.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -52,13 +52,13 @@
 
 * https://pypi.python.org/pypi/cffi
 
-* Checksums of the "source" package version 1.13.0:
+* Checksums of the "source" package version 1.13.1:
 
-   - MD5: 2c90be6d3bfbf80ab732b28fd677051d
+   - MD5: ...
 
-   - SHA: 75977bbad8ea8c15f2c34edde5efcf39462ea43e
+   - SHA: ...
 
-   - SHA256: 8fe230f612c18af1df6f348d02d682fe2c28ca0a6c3856c99599cdacae7cf226
+   - SHA256: ...
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -3,6 +3,25 @@
 ======================
 
 
+v1.13.1
+=======
+
+* deprecate the way to declare in ``cdef()`` a global variable with only
+  ``void *foo;``.  You should always use a storage class, like ``extern void
+  *foo;`` or maybe ``static void *foo;``.  These are all equivalent for
+  the purposes of ``cdef()``, but the reason for deprecating the bare version
+  is that (as far as I know) it would always be mistake in a real C header.
+
+* fix the regression ``RuntimeError: found a situation in which we try
+  to build a type recursively`` (`issue #429`_).
+
+* fixed `issue #427`_ where a multithreading mistake in the embedding logic
+  initialization code would cause deadlocks on CPython 3.7.
+
+.. _`issue #429`: https://bitbucket.org/cffi/cffi/issues/429/
+.. _`issue #427`: https://bitbucket.org/cffi/cffi/issues/427/
+
+
 v1.13
 =====
 
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -198,7 +198,7 @@
 
 `Mailing list <https://groups.google.com/forum/#!forum/python-cffi>`_
 """,
-        version='1.13.0',
+        version='1.13.1',
         packages=['cffi'] if cpython else [],
         package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h', 
                                '_embedding.h', '_cffi_errors.h']}
diff --git a/testing/cffi0/test_function.py b/testing/cffi0/test_function.py
--- a/testing/cffi0/test_function.py
+++ b/testing/cffi0/test_function.py
@@ -113,7 +113,7 @@
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
             int fputs(const char *, void *);
-            void *stderr;
+            extern void *stderr;
         """)
         needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
@@ -130,7 +130,7 @@
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
             int fputs(char *, void *);
-            void *stderr;
+            extern void *stderr;
         """)
         needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
@@ -147,7 +147,7 @@
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
            int fprintf(void *, const char *format, ...);
-           void *stderr;
+           extern void *stderr;
         """)
         needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
@@ -209,7 +209,7 @@
             py.test.skip("probably no symbol 'stderr' in the lib")
         ffi.cdef("""
             int fputs(const char *, void *);
-            void *stderr;
+            extern void *stderr;
         """)
         needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
@@ -256,7 +256,7 @@
             py.test.skip("probably no symbol 'stdout' in the lib")
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
-            void *stdout;
+            extern void *stdout;
         """)
         needs_dlopen_none()
         C = ffi.dlopen(None)
@@ -496,7 +496,7 @@
         ffi.cdef("""
             typedef enum { MYE1, MYE2 } myenum_t;
             double myfunc(double);
-            double myvar;
+            extern double myvar;
             const double myconst;
             #define MYFOO 42
         """)
@@ -507,7 +507,7 @@
         if self.Backend is CTypesBackend:
             py.test.skip("not with the ctypes backend")
         ffi = FFI(backend=self.Backend())
-        ffi.cdef("int foobar(void); int foobaz;")
+        ffi.cdef("int foobar(void); extern int foobaz;")
         lib = ffi.dlopen(lib_m)
         ffi.dlclose(lib)
         e = py.test.raises(ValueError, getattr, lib, 'foobar')
diff --git a/testing/cffi0/test_ownlib.py b/testing/cffi0/test_ownlib.py
--- a/testing/cffi0/test_ownlib.py
+++ b/testing/cffi0/test_ownlib.py
@@ -201,7 +201,7 @@
             py.test.skip("fix the auto-generation of the tiny test lib")
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
-            int my_array[7];
+            extern int my_array[7];
         """)
         ownlib = ffi.dlopen(self.module)
         for i in range(7):
@@ -223,7 +223,7 @@
             py.test.skip("not supported by the ctypes backend")
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
-            int my_array[];
+            extern int my_array[];
         """)
         ownlib = ffi.dlopen(self.module)
         for i in range(7):
@@ -291,7 +291,7 @@
                 long bottom;
             } RECT;
             
-            long left, top, right, bottom;
+            extern long left, top, right, bottom;
 
             RECT ReturnRect(int i, RECT ar, RECT* br, POINT cp, RECT dr,
                         RECT *er, POINT fp, RECT gr);
@@ -321,7 +321,7 @@
         if self.Backend is CTypesBackend:
             py.test.skip("not implemented with the ctypes backend")
         ffi = FFI(backend=self.Backend())
-        ffi.cdef("long left; int test_getting_errno(void);")
+        ffi.cdef("extern long left; int test_getting_errno(void);")
         lib = ffi.dlopen(self.module)
         lib.left = 123456
         p = ffi.addressof(lib, "left")
diff --git a/testing/cffi0/test_parsing.py b/testing/cffi0/test_parsing.py
--- a/testing/cffi0/test_parsing.py
+++ b/testing/cffi0/test_parsing.py
@@ -324,6 +324,7 @@
     assert value == sys.maxsize * 2 - 40
 
 def test__is_constant_globalvar():
+    import warnings
     for input, expected_output in [
         ("int a;",          False),
         ("const int a;",    True),
@@ -341,10 +342,13 @@
         ("const int a[5][6];", False),
         ]:
         ffi = FFI()
-        ffi.cdef(input)
+        with warnings.catch_warnings(record=True) as log:
+            warnings.simplefilter("always")
+            ffi.cdef(input)
         declarations = ffi._parser._declarations
         assert ('constant a' in declarations) == expected_output
         assert ('variable a' in declarations) == (not expected_output)
+        assert len(log) == (1 - expected_output)
 
 def test_restrict():
     from cffi import model
@@ -354,7 +358,7 @@
         ("int *a;",            False),
         ]:
         ffi = FFI()
-        ffi.cdef(input)
+        ffi.cdef("extern " + input)
         tp, quals = ffi._parser._declarations['variable a']
         assert bool(quals & model.Q_RESTRICT) == expected_output
 
diff --git a/testing/cffi0/test_verify.py b/testing/cffi0/test_verify.py
--- a/testing/cffi0/test_verify.py
+++ b/testing/cffi0/test_verify.py
@@ -286,7 +286,7 @@
 def test_var_signed_integer_types():
     ffi = FFI()
     lst = all_signed_integer_types(ffi)
-    csource = "\n".join(["%s somevar_%s;" % (tp, tp.replace(' ', '_'))
+    csource = "\n".join(["static %s somevar_%s;" % (tp, tp.replace(' ', '_'))
                          for tp in lst])
     ffi.cdef(csource)
     lib = ffi.verify(csource)
@@ -305,7 +305,7 @@
 def test_var_unsigned_integer_types():
     ffi = FFI()
     lst = all_unsigned_integer_types(ffi)
-    csource = "\n".join(["%s somevar_%s;" % (tp, tp.replace(' ', '_'))
+    csource = "\n".join(["static %s somevar_%s;" % (tp, tp.replace(' ', '_'))
                          for tp in lst])
     ffi.cdef(csource)
     lib = ffi.verify(csource)
@@ -817,8 +817,8 @@
 
 def test_access_variable():
     ffi = FFI()
-    ffi.cdef("int foo(void);\n"
-             "int somenumber;")
+    ffi.cdef("static int foo(void);\n"
+             "static int somenumber;")
     lib = ffi.verify("""
         static int somenumber = 2;
         static int foo(void) {
@@ -835,7 +835,7 @@
 def test_access_address_of_variable():
     # access the address of 'somenumber': need a trick
     ffi = FFI()
-    ffi.cdef("int somenumber; static int *const somenumberptr;")
+    ffi.cdef("static int somenumber; static int *const somenumberptr;")
     lib = ffi.verify("""
         static int somenumber = 2;
         #define somenumberptr (&somenumber)
@@ -848,7 +848,7 @@
 def test_access_array_variable(length=5):
     ffi = FFI()
     ffi.cdef("int foo(int);\n"
-             "int somenumber[%s];" % (length,))
+             "static int somenumber[%s];" % (length,))
     lib = ffi.verify("""
         static int somenumber[] = {2, 2, 3, 4, 5};
         static int foo(int i) {
@@ -880,7 +880,7 @@
     ffi = FFI()
     ffi.cdef("struct foo { int x; ...; };\n"
              "int foo(int);\n"
-             "struct foo stuff;")
+             "static struct foo stuff;")
     lib = ffi.verify("""
         struct foo { int x, y, z; };
         static struct foo stuff = {2, 5, 8};
@@ -904,9 +904,9 @@
 
 def test_access_callback():
     ffi = FFI()
-    ffi.cdef("int (*cb)(int);\n"
-             "int foo(int);\n"
-             "void reset_cb(void);")
+    ffi.cdef("static int (*cb)(int);\n"
+             "static int foo(int);\n"
+             "static void reset_cb(void);")
     lib = ffi.verify("""
         static int g(int x) { return x * 7; }
         static int (*cb)(int);
@@ -922,9 +922,9 @@
 def test_access_callback_function_typedef():
     ffi = FFI()
     ffi.cdef("typedef int mycallback_t(int);\n"
-             "mycallback_t *cb;\n"
-             "int foo(int);\n"
-             "void reset_cb(void);")
+             "static mycallback_t *cb;\n"
+             "static int foo(int);\n"
+             "static void reset_cb(void);")
     lib = ffi.verify("""
         static int g(int x) { return x * 7; }
         static int (*cb)(int);
@@ -1074,7 +1074,7 @@
 def test_autofilled_struct_as_argument_dynamic():
     ffi = FFI()
     ffi.cdef("struct foo_s { long a; ...; };\n"
-             "int (*foo)(struct foo_s);")
+             "static int (*foo)(struct foo_s);")
     lib = ffi.verify("""
         struct foo_s {
             double b;
@@ -1083,7 +1083,7 @@
         int foo1(struct foo_s s) {
             return (int)s.a - (int)s.b;
         }
-        int (*foo)(struct foo_s s) = &foo1;
+        static int (*foo)(struct foo_s s) = &foo1;
     """)
     e = py.test.raises(NotImplementedError, lib.foo, "?")
     msg = ("ctype 'struct foo_s' not supported as argument.  It is a struct "
@@ -1453,7 +1453,7 @@
         py.test.skip("_Bool not in MSVC")
     ffi = FFI()
     ffi.cdef("struct foo_s { _Bool x; };"
-             "_Bool foo(_Bool); _Bool (*foop)(_Bool);")
+             "_Bool foo(_Bool); static _Bool (*foop)(_Bool);")
     lib = ffi.verify("""
         struct foo_s { _Bool x; };
         int foo(int arg) {
@@ -1462,7 +1462,7 @@
         _Bool _foofunc(_Bool x) {
             return !x;
         }
-        _Bool (*foop)(_Bool) = _foofunc;
+        static _Bool (*foop)(_Bool) = _foofunc;
     """)
     p = ffi.new("struct foo_s *")
     p.x = 1
@@ -1653,7 +1653,7 @@
 
 def test_FILE_stored_explicitly():
     ffi = FFI()
-    ffi.cdef("int myprintf11(const char *, int); FILE *myfile;")
+    ffi.cdef("int myprintf11(const char *, int); extern FILE *myfile;")
     lib = ffi.verify("""
         #include <stdio.h>
         FILE *myfile;
@@ -1679,19 +1679,19 @@
 
 def test_global_array_with_missing_length():
     ffi = FFI()
-    ffi.cdef("int fooarray[];")
+    ffi.cdef("extern int fooarray[];")
     lib = ffi.verify("int fooarray[50];")
     assert repr(lib.fooarray).startswith("<cdata 'int *'")
 
 def test_global_array_with_dotdotdot_length():
     ffi = FFI()
-    ffi.cdef("int fooarray[...];")
+    ffi.cdef("extern int fooarray[...];")
     lib = ffi.verify("int fooarray[50];")
     assert repr(lib.fooarray).startswith("<cdata 'int[50]'")
 
 def test_bad_global_array_with_dotdotdot_length():
     ffi = FFI()
-    ffi.cdef("int fooarray[...];")
+    ffi.cdef("extern int fooarray[...];")
     py.test.raises(VerificationError, ffi.verify, "char fooarray[23];")
 
 def test_struct_containing_struct():
@@ -1812,7 +1812,7 @@
 def test_callback_indirection():
     ffi = FFI()
     ffi.cdef("""
-        int (*python_callback)(int how_many, int *values);
+        static int (*python_callback)(int how_many, int *values);
         int (*const c_callback)(int,...);   /* pass this ptr to C routines */
         int some_c_function(int(*cb)(int,...));
     """)
@@ -1946,24 +1946,24 @@
 
 def test_bug_const_char_ptr_array_1():
     ffi = FFI()
-    ffi.cdef("""const char *a[...];""")
+    ffi.cdef("""extern const char *a[...];""")
     lib = ffi.verify("""const char *a[5];""")
     assert repr(ffi.typeof(lib.a)) == "<ctype 'char *[5]'>"
 
 def test_bug_const_char_ptr_array_2():
     from cffi import FFI     # ignore warnings
     ffi = FFI()
-    ffi.cdef("""const int a[];""")
+    ffi.cdef("""extern const int a[];""")
     lib = ffi.verify("""const int a[5];""")
     assert repr(ffi.typeof(lib.a)) == "<ctype 'int *'>"
 
 def _test_various_calls(force_libffi):
     cdef_source = """
-    int xvalue;
-    long long ivalue, rvalue;
-    float fvalue;
-    double dvalue;
-    long double Dvalue;
+    extern int xvalue;
+    extern long long ivalue, rvalue;
+    extern float fvalue;
+    extern double dvalue;
+    extern long double Dvalue;
     signed char tf_bb(signed char x, signed char c);
     unsigned char tf_bB(signed char x, unsigned char c);
     short tf_bh(signed char x, short c);
@@ -2147,7 +2147,7 @@
     # exported symbols as well.  So we must not export a simple name
     # like 'foo'!
     ffi1 = FFI()
-    ffi1.cdef("int foo_verify_dlopen_flags;")
+    ffi1.cdef("extern int foo_verify_dlopen_flags;")
 
     lib1 = ffi1.verify("int foo_verify_dlopen_flags;",
                        flags=ffi1.RTLD_GLOBAL | ffi1.RTLD_LAZY)
@@ -2161,7 +2161,7 @@
 def get_second_lib():
     # Hack, using modulename makes the test fail
     ffi2 = FFI()
-    ffi2.cdef("int foo_verify_dlopen_flags;")
+    ffi2.cdef("extern int foo_verify_dlopen_flags;")
     lib2 = ffi2.verify("int foo_verify_dlopen_flags;",
                        flags=ffi2.RTLD_GLOBAL | ffi2.RTLD_LAZY)
     return lib2
diff --git a/testing/cffi1/test_dlopen.py b/testing/cffi1/test_dlopen.py
--- a/testing/cffi1/test_dlopen.py
+++ b/testing/cffi1/test_dlopen.py
@@ -6,7 +6,7 @@
 
 def test_simple():
     ffi = FFI()
-    ffi.cdef("int close(int); static const int BB = 42; int somevar;")
+    ffi.cdef("int close(int); static const int BB = 42; extern int somevar;")
     target = udir.join('test_simple.py')
     make_py_source(ffi, 'test_simple', str(target))
     assert target.read() == r"""# auto-generated file
@@ -196,7 +196,7 @@
 
 def test_global_var():
     ffi = FFI()
-    ffi.cdef("int myglob;")
+    ffi.cdef("extern int myglob;")
     target = udir.join('test_global_var.py')
     make_py_source(ffi, 'test_global_var', str(target))
     assert target.read() == r"""# auto-generated file
diff --git a/testing/cffi1/test_new_ffi_1.py b/testing/cffi1/test_new_ffi_1.py
--- a/testing/cffi1/test_new_ffi_1.py
+++ b/testing/cffi1/test_new_ffi_1.py
@@ -1779,7 +1779,7 @@
 
     def test_import_from_lib(self):
         ffi2 = cffi.FFI()
-        ffi2.cdef("int myfunc(int); int myvar;\n#define MYFOO ...\n")
+        ffi2.cdef("int myfunc(int); extern int myvar;\n#define MYFOO ...\n")
         outputfilename = recompile(ffi2, "_test_import_from_lib",
                                    "int myfunc(int x) { return x + 1; }\n"
                                    "int myvar = -5;\n"
diff --git a/testing/cffi1/test_re_python.py b/testing/cffi1/test_re_python.py
--- a/testing/cffi1/test_re_python.py
+++ b/testing/cffi1/test_re_python.py
@@ -63,11 +63,11 @@
     #define BIGNEG -420000000000L
     int add42(int);
     int add43(int, ...);
-    int globalvar42;
+    extern int globalvar42;
     const int globalconst42;
     const char *const globalconsthello;
     int no_such_function(int);
-    int no_such_globalvar;
+    extern int no_such_globalvar;
     struct foo_s;
     typedef struct bar_s { int x; signed char a[]; } bar_t;
     enum foo_e { AA, BB, CC };
@@ -75,6 +75,7 @@
     struct with_union { union { int a; char b; }; };
     union with_struct { struct { int a; char b; }; };
     struct NVGcolor { union { float rgba[4]; struct { float r,g,b,a; }; }; };
+    typedef struct selfref { struct selfref *next; } *selfref_ptr_t;
     """)
     ffi.set_source('re_python_pysrc', None)
     ffi.emit_python_code(str(tmpdir.join('re_python_pysrc.py')))
@@ -254,3 +255,8 @@
     assert ffi.offsetof("struct NVGcolor", "g") == FLOAT
     assert ffi.offsetof("struct NVGcolor", "b") == FLOAT * 2
     assert ffi.offsetof("struct NVGcolor", "a") == FLOAT * 3
+
+def test_selfref():
+    # based on issue #429
+    from re_python_pysrc import ffi
+    ffi.new("selfref_ptr_t")
diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -83,7 +83,7 @@
                      "(FUNCTION 1)(PRIMITIVE 7)(FUNCTION_END 1)(POINTER 0)")
 
 def test_type_table_array():
-    check_type_table("int a[100];",
+    check_type_table("extern int a[100];",
                      "(PRIMITIVE 7)(ARRAY 0)(None 100)")
 
 def test_type_table_typedef():
@@ -158,7 +158,7 @@
 
 def test_global_var_array():
     ffi = FFI()
-    ffi.cdef("int a[100];")
+    ffi.cdef("extern int a[100];")
     lib = verify(ffi, 'test_global_var_array', 'int a[100] = { 9999 };')
     lib.a[42] = 123456
     assert lib.a[42] == 123456
@@ -182,7 +182,7 @@
 
 def test_global_var_int():
     ffi = FFI()
-    ffi.cdef("int a, b, c;")
+    ffi.cdef("extern int a, b, c;")
     lib = verify(ffi, 'test_global_var_int', 'int a = 999, b, c;')
     assert lib.a == 999
     lib.a -= 1001
@@ -283,7 +283,7 @@
 
 def test_dir():
     ffi = FFI()
-    ffi.cdef("int ff(int); int aa; static const int my_constant;")
+    ffi.cdef("int ff(int); extern int aa; static const int my_constant;")
     lib = verify(ffi, 'test_dir', """
         #define my_constant  (-45)
         int aa;
@@ -405,7 +405,7 @@
 
 def test_dotdotdot_global_array():
     ffi = FFI()
-    ffi.cdef("int aa[...]; int bb[...];")
+    ffi.cdef("extern int aa[...]; extern int bb[...];")
     lib = verify(ffi, 'test_dotdotdot_global_array',
                  "int aa[41]; int bb[12];")
     assert ffi.sizeof(lib.aa) == 41 * 4
@@ -560,37 +560,37 @@
 
 def test_bad_size_of_global_1():
     ffi = FFI()
-    ffi.cdef("short glob;")
+    ffi.cdef("extern short glob;")
     py.test.raises(VerificationError, verify, ffi,
                    "test_bad_size_of_global_1", "long glob;")
 
 def test_bad_size_of_global_2():
     ffi = FFI()
-    ffi.cdef("int glob[10];")
+    ffi.cdef("extern int glob[10];")
     py.test.raises(VerificationError, verify, ffi,
                    "test_bad_size_of_global_2", "int glob[9];")
 
 def test_unspecified_size_of_global_1():
     ffi = FFI()
-    ffi.cdef("int glob[];")
+    ffi.cdef("extern int glob[];")
     lib = verify(ffi, "test_unspecified_size_of_global_1", "int glob[10];")
     assert ffi.typeof(lib.glob) == ffi.typeof("int *")
 
 def test_unspecified_size_of_global_2():
     ffi = FFI()
-    ffi.cdef("int glob[][5];")
+    ffi.cdef("extern int glob[][5];")
     lib = verify(ffi, "test_unspecified_size_of_global_2", "int glob[10][5];")
     assert ffi.typeof(lib.glob) == ffi.typeof("int(*)[5]")
 
 def test_unspecified_size_of_global_3():
     ffi = FFI()
-    ffi.cdef("int glob[][...];")
+    ffi.cdef("extern int glob[][...];")
     lib = verify(ffi, "test_unspecified_size_of_global_3", "int glob[10][5];")
     assert ffi.typeof(lib.glob) == ffi.typeof("int(*)[5]")
 
 def test_unspecified_size_of_global_4():
     ffi = FFI()
-    ffi.cdef("int glob[...][...];")
+    ffi.cdef("extern int glob[...][...];")
     lib = verify(ffi, "test_unspecified_size_of_global_4", "int glob[10][5];")
     assert ffi.typeof(lib.glob) == ffi.typeof("int[10][5]")
 
@@ -813,7 +813,7 @@
 def test_address_of_global_var():
     ffi = FFI()
     ffi.cdef("""
-        long bottom, bottoms[2];
+        extern long bottom, bottoms[2];
         long FetchRectBottom(void);
         long FetchRectBottoms1(void);
         #define FOOBAR 42
@@ -968,7 +968,7 @@
     ffi = FFI()
     ffi.cdef("""
         typedef ... opaque_t;
-        opaque_t globvar;
+        extern opaque_t globvar;
     """)
     lib = verify(ffi, 'test_variable_of_unknown_size', """
         typedef char opaque_t[6];
@@ -1013,7 +1013,7 @@
 def test_call_with_incomplete_structs():
     ffi = FFI()
     ffi.cdef("typedef struct {...;} foo_t; "
-             "foo_t myglob; "
+             "extern foo_t myglob; "
              "foo_t increment(foo_t s); "
              "double getx(foo_t s);")
     lib = verify(ffi, 'test_call_with_incomplete_structs', """
@@ -1057,7 +1057,7 @@
 
 def test_global_var_array_2():
     ffi = FFI()
-    ffi.cdef("int a[...][...];")
+    ffi.cdef("extern int a[...][...];")
     lib = verify(ffi, 'test_global_var_array_2', 'int a[10][8];')
     lib.a[9][7] = 123456
     assert lib.a[9][7] == 123456
@@ -1070,7 +1070,7 @@
 
 def test_global_var_array_3():
     ffi = FFI()
-    ffi.cdef("int a[][...];")
+    ffi.cdef("extern int a[][...];")
     lib = verify(ffi, 'test_global_var_array_3', 'int a[10][8];')
     lib.a[9][7] = 123456
     assert lib.a[9][7] == 123456
@@ -1081,7 +1081,7 @@
 
 def test_global_var_array_4():
     ffi = FFI()
-    ffi.cdef("int a[10][...];")
+    ffi.cdef("extern int a[10][...];")
     lib = verify(ffi, 'test_global_var_array_4', 'int a[10][8];')
     lib.a[9][7] = 123456
     assert lib.a[9][7] == 123456
@@ -1204,7 +1204,7 @@
 
 def test_import_from_lib():
     ffi = FFI()
-    ffi.cdef("int mybar(int); int myvar;\n#define MYFOO ...")
+    ffi.cdef("int mybar(int); static int myvar;\n#define MYFOO ...")
     lib = verify(ffi, 'test_import_from_lib',
                  "#define MYFOO 42\n"
                  "static int mybar(int x) { return x + 1; }\n"
@@ -1220,7 +1220,7 @@
 
 def test_macro_var_callback():
     ffi = FFI()
-    ffi.cdef("int my_value; int *(*get_my_value)(void);")
+    ffi.cdef("extern int my_value; extern int *(*get_my_value)(void);")
     lib = verify(ffi, 'test_macro_var_callback',
                  "int *(*get_my_value)(void);\n"
                  "#define my_value (*get_my_value())")
@@ -1335,7 +1335,7 @@
 
 def test_const_function_type_args():
     ffi = FFI()
-    ffi.cdef("""int (*foobar)(const int a, const int *b, const int c[]);""")
+    ffi.cdef("""extern int(*foobar)(const int a,const int*b,const int c[]);""")
     lib = verify(ffi, 'test_const_function_type_args', """
         int (*foobar)(const int a, const int *b, const int c[]);
     """)
@@ -1625,7 +1625,7 @@
 
 def test_extern_python_bogus_name():
     ffi = FFI()
-    ffi.cdef("int abc;")
+    ffi.cdef("extern int abc;")
     lib = verify(ffi, 'test_extern_python_bogus_name', "int abc;")
     def fn():
         pass
@@ -1786,8 +1786,8 @@
     ffi.cdef("""
         extern "Python" int __stdcall foo(int);
         extern "Python" int WINAPI bar(int);
-        int (__stdcall * mycb1)(int);
-        int indirect_call(int);
+        static int (__stdcall * mycb1)(int);
+        static int indirect_call(int);
     """)
     lib = verify(ffi, 'test_extern_python_stdcall', """
         #ifndef _MSC_VER
@@ -1855,7 +1855,7 @@
 
 def test_introspect_global_var():
     ffi = FFI()
-    ffi.cdef("float g1;")
+    ffi.cdef("extern float g1;")
     lib = verify(ffi, 'test_introspect_global_var', """
         float g1;
     """)
@@ -1866,7 +1866,7 @@
 
 def test_introspect_global_var_array():
     ffi = FFI()
-    ffi.cdef("float g1[100];")
+    ffi.cdef("extern float g1[100];")
     lib = verify(ffi, 'test_introspect_global_var_array', """
         float g1[100];
     """)
@@ -2089,7 +2089,7 @@
     ffi = FFI()
     ffi.cdef("""
         typedef int foo_t[...], bar_t[...];
-        int gv[...];
+        extern int gv[...];
         typedef int mat_t[...][...];
         typedef int vmat_t[][...];
         """)
diff --git a/testing/cffi1/test_verify1.py b/testing/cffi1/test_verify1.py
--- a/testing/cffi1/test_verify1.py
+++ b/testing/cffi1/test_verify1.py
@@ -267,7 +267,7 @@
 def test_var_signed_integer_types():
     ffi = FFI()
     lst = all_signed_integer_types(ffi)
-    csource = "\n".join(["%s somevar_%s;" % (tp, tp.replace(' ', '_'))
+    csource = "\n".join(["static %s somevar_%s;" % (tp, tp.replace(' ', '_'))
                          for tp in lst])
     ffi.cdef(csource)
     lib = ffi.verify(csource)
@@ -286,7 +286,7 @@
 def test_var_unsigned_integer_types():
     ffi = FFI()
     lst = all_unsigned_integer_types(ffi)
-    csource = "\n".join(["%s somevar_%s;" % (tp, tp.replace(' ', '_'))
+    csource = "\n".join(["static %s somevar_%s;" % (tp, tp.replace(' ', '_'))
                          for tp in lst])
     ffi.cdef(csource)
     lib = ffi.verify(csource)
@@ -790,8 +790,8 @@
 
 def test_access_variable():
     ffi = FFI()
-    ffi.cdef("int foo(void);\n"
-             "int somenumber;")
+    ffi.cdef("static int foo(void);\n"
+             "static int somenumber;")
     lib = ffi.verify("""
         static int somenumber = 2;
         static int foo(void) {
@@ -808,7 +808,7 @@
 def test_access_address_of_variable():
     # access the address of 'somenumber': need a trick
     ffi = FFI()
-    ffi.cdef("int somenumber; static int *const somenumberptr;")
+    ffi.cdef("static int somenumber; static int *const somenumberptr;")
     lib = ffi.verify("""
         static int somenumber = 2;
         #define somenumberptr (&somenumber)
@@ -820,8 +820,8 @@
 
 def test_access_array_variable(length=5):
     ffi = FFI()
-    ffi.cdef("int foo(int);\n"
-             "int somenumber[%s];" % (length,))
+    ffi.cdef("static int foo(int);\n"
+             "static int somenumber[%s];" % (length,))
     lib = ffi.verify("""
         static int somenumber[] = {2, 2, 3, 4, 5};
         static int foo(int i) {
@@ -852,8 +852,8 @@
 def test_access_struct_variable():
     ffi = FFI()
     ffi.cdef("struct foo { int x; ...; };\n"
-             "int foo(int);\n"
-             "struct foo stuff;")
+             "static int foo(int);\n"
+             "static struct foo stuff;")
     lib = ffi.verify("""
         struct foo { int x, y, z; };
         static struct foo stuff = {2, 5, 8};
@@ -877,9 +877,9 @@
 
 def test_access_callback():
     ffi = FFI()
-    ffi.cdef("int (*cb)(int);\n"
-             "int foo(int);\n"
-             "void reset_cb(void);")
+    ffi.cdef("static int (*cb)(int);\n"
+             "static int foo(int);\n"
+             "static void reset_cb(void);")
     lib = ffi.verify("""
         static int g(int x) { return x * 7; }
         static int (*cb)(int);
@@ -895,9 +895,9 @@
 def test_access_callback_function_typedef():
     ffi = FFI()
     ffi.cdef("typedef int mycallback_t(int);\n"
-             "mycallback_t *cb;\n"
-             "int foo(int);\n"
-             "void reset_cb(void);")
+             "static mycallback_t *cb;\n"
+             "static int foo(int);\n"
+             "static void reset_cb(void);")
     lib = ffi.verify("""
         static int g(int x) { return x * 7; }
         static int (*cb)(int);
@@ -1038,7 +1038,7 @@
 def test_autofilled_struct_as_argument_dynamic():
     ffi = FFI()
     ffi.cdef("struct foo_s { long a; ...; };\n"
-             "int (*foo)(struct foo_s);")
+             "static int (*foo)(struct foo_s);")
     lib = ffi.verify("""
         struct foo_s {
             double b;
@@ -1047,7 +1047,7 @@
         int foo1(struct foo_s s) {
             return (int)s.a - (int)s.b;
         }
-        int (*foo)(struct foo_s s) = &foo1;
+        static int (*foo)(struct foo_s s) = &foo1;
     """)
     e = py.test.raises(NotImplementedError, lib.foo, "?")
     msg = ("ctype 'struct foo_s' not supported as argument.  It is a struct "
@@ -1423,7 +1423,7 @@
         py.test.skip("_Bool not in MSVC")
     ffi = FFI()
     ffi.cdef("struct foo_s { _Bool x; };"
-             "_Bool foo(_Bool); _Bool (*foop)(_Bool);")
+             "_Bool foo(_Bool); static _Bool (*foop)(_Bool);")
     lib = ffi.verify("""
         struct foo_s { _Bool x; };
         int foo(int arg) {
@@ -1432,7 +1432,7 @@
         _Bool _foofunc(_Bool x) {
             return !x;
         }
-        _Bool (*foop)(_Bool) = _foofunc;
+        static _Bool (*foop)(_Bool) = _foofunc;
     """)
     p = ffi.new("struct foo_s *")
     p.x = 1
@@ -1617,7 +1617,7 @@
 
 def test_FILE_stored_explicitly():
     ffi = FFI()
-    ffi.cdef("int myprintf11(const char *, int); FILE *myfile;")
+    ffi.cdef("int myprintf11(const char *, int); extern FILE *myfile;")
     lib = ffi.verify("""
         #include <stdio.h>
         FILE *myfile;
@@ -1643,13 +1643,13 @@
 
 def test_global_array_with_missing_length():
     ffi = FFI()
-    ffi.cdef("int fooarray[];")
+    ffi.cdef("extern int fooarray[];")
     lib = ffi.verify("int fooarray[50];")
     assert repr(lib.fooarray).startswith("<cdata 'int *'")
 
 def test_global_array_with_dotdotdot_length():
     ffi = FFI()
-    ffi.cdef("int fooarray[...];")
+    ffi.cdef("extern int fooarray[...];")
     lib = ffi.verify("int fooarray[50];")
     assert repr(lib.fooarray).startswith("<cdata 'int[50]'")
 
@@ -1657,7 +1657,7 @@
     py.test.xfail("was detected only because 23 bytes cannot be divided by 4; "
                   "redo more generally")
     ffi = FFI()
-    ffi.cdef("int fooarray[...];")
+    ffi.cdef("extern int fooarray[...];")
     py.test.raises(VerificationError, ffi.verify, "char fooarray[23];")
 
 def test_struct_containing_struct():
@@ -1778,7 +1778,7 @@
 def test_callback_indirection():
     ffi = FFI()
     ffi.cdef("""
-        int (*python_callback)(int how_many, int *values);
+        static int (*python_callback)(int how_many, int *values);
         int (*const c_callback)(int,...);   /* pass this ptr to C routines */
         int some_c_function(int(*cb)(int,...));
     """)
@@ -1912,23 +1912,23 @@
 
 def test_bug_const_char_ptr_array_1():
     ffi = FFI()
-    ffi.cdef("""const char *a[...];""")
+    ffi.cdef("""extern const char *a[...];""")
     lib = ffi.verify("""const char *a[5];""")
     assert repr(ffi.typeof(lib.a)) == "<ctype 'char *[5]'>"
 
 def test_bug_const_char_ptr_array_2():
     ffi = FFI()
-    ffi.cdef("""const int a[];""")
+    ffi.cdef("""extern const int a[];""")
     lib = ffi.verify("""const int a[5];""")
     assert repr(ffi.typeof(lib.a)) == "<ctype 'int *'>"
 
 def _test_various_calls(force_libffi):
     cdef_source = """
-    int xvalue;
-    long long ivalue, rvalue;
-    float fvalue;
-    double dvalue;
-    long double Dvalue;
+    extern int xvalue;
+    extern long long ivalue, rvalue;
+    extern float fvalue;
+    extern double dvalue;
+    extern long double Dvalue;
     signed char tf_bb(signed char x, signed char c);
     unsigned char tf_bB(signed char x, unsigned char c);
     short tf_bh(signed char x, short c);
@@ -2111,7 +2111,7 @@
     old = sys.getdlopenflags()
     try:
         ffi1 = FFI()
-        ffi1.cdef("int foo_verify_dlopen_flags_1;")
+        ffi1.cdef("extern int foo_verify_dlopen_flags_1;")
         sys.setdlopenflags(ffi1.RTLD_GLOBAL | ffi1.RTLD_NOW)
         lib1 = ffi1.verify("int foo_verify_dlopen_flags_1;")
     finally:
@@ -2252,7 +2252,7 @@
 
 def test_macro_var():
     ffi = FFI()
-    ffi.cdef("int myarray[50], my_value;")
+    ffi.cdef("extern int myarray[50], my_value;")
     lib = ffi.verify("""
         int myarray[50];
         int *get_my_value(void) {
diff --git a/testing/embedding/add1.py b/testing/embedding/add1.py
--- a/testing/embedding/add1.py
+++ b/testing/embedding/add1.py
@@ -11,7 +11,11 @@
     sys.stdout.write("preparing")
     for i in range(3):
         sys.stdout.flush()
-        time.sleep(0.2)
+        # Windows: sometimes time.sleep() doesn't sleep at all.
+        # This appears to occur on recent versions of python only.
+        t_end = time.time() + 0.19
+        while time.time() < t_end:
+            time.sleep(0.2)
         sys.stdout.write(".")
     sys.stdout.write("\n")
 
diff --git a/testing/embedding/add_recursive.py b/testing/embedding/add_recursive.py
--- a/testing/embedding/add_recursive.py
+++ b/testing/embedding/add_recursive.py
@@ -3,7 +3,7 @@
 ffi = cffi.FFI()
 
 ffi.embedding_api("""
-    int (*my_callback)(int);
+    extern int (*my_callback)(int);
     int add_rec(int, int);
 """)
 
diff --git a/testing/embedding/test_thread.py b/testing/embedding/test_thread.py
--- a/testing/embedding/test_thread.py
+++ b/testing/embedding/test_thread.py
@@ -21,17 +21,21 @@
         add1_cffi = self.prepare_module('add1')
         add2_cffi = self.prepare_module('add2')
         self.compile('thread2-test', [add1_cffi, add2_cffi], threads=True)
-        output = self.execute('thread2-test')
-        output = self._take_out(output, "preparing")
-        output = self._take_out(output, ".")
-        output = self._take_out(output, ".")
-        # at least the 3rd dot should be after everything from ADD2
-        assert output == ("starting\n"
-                          "prepADD2\n"
-                          "adding 1000 and 200 and 30\n"
-                          ".\n"
-                          "adding 40 and 2\n"
-                          "done\n")
+        for i in range(3):
+            output = self.execute('thread2-test')
+            print('='*79)
+            print(output)
+            print('='*79)
+            output = self._take_out(output, "preparing")
+            output = self._take_out(output, ".")
+            output = self._take_out(output, ".")
+            # at least the 3rd dot should be after everything from ADD2
+            assert output == ("starting\n"
+                              "prepADD2\n"
+                              "adding 1000 and 200 and 30\n"
+                              ".\n"
+                              "adding 40 and 2\n"
+                              "done\n")
 
     def test_alt_issue(self):
         add1_cffi = self.prepare_module('add1')

From pypy.commits at gmail.com  Mon Oct 21 10:57:55 2019
From: pypy.commits at gmail.com (arigo)
Date: Mon, 21 Oct 2019 07:57:55 -0700 (PDT)
Subject: [pypy-commit] cffi release-1.13: md5/sha
Message-ID: <5dadc773.1c69fb81.b296.62b1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-1.13
Changeset: r3305:9cc790d05bc5
Date: 2019-10-21 16:57 +0200
http://bitbucket.org/cffi/cffi/changeset/9cc790d05bc5/

Log:	md5/sha

diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -54,11 +54,11 @@
 
 * Checksums of the "source" package version 1.13.1:
 
-   - MD5: ...
+   - MD5: 824ad9f228fbc6ce203c334e2ff4ab8f
 
-   - SHA: ...
+   - SHA: 678dad3a3102ebcbeab8f5f132968afa444f6a54
 
-   - SHA256: ...
+   - SHA256: 558b3afef987cf4b17abd849e7bedf64ee12b28175d564d05b628a0f9355599b
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``

From pypy.commits at gmail.com  Mon Oct 21 10:57:56 2019
From: pypy.commits at gmail.com (arigo)
Date: Mon, 21 Oct 2019 07:57:56 -0700 (PDT)
Subject: [pypy-commit] cffi release-1.13: Added tag v1.13.1 for changeset
 9cc790d05bc5
Message-ID: <5dadc774.1c69fb81.18d04.00b8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-1.13
Changeset: r3306:9114b3a1924f
Date: 2019-10-21 16:57 +0200
http://bitbucket.org/cffi/cffi/changeset/9114b3a1924f/

Log:	Added tag v1.13.1 for changeset 9cc790d05bc5

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -22,3 +22,4 @@
 e0c76668ecf5ab1ad005799cfe1b7cb58610f155 v1.12.2
 bf80722dea36237710083315e336c81bc8571fd0 v1.12.3
 17c06e75f54cf5a31865961f0b498f99232a0bc8 v1.13.0
+9cc790d05bc5256eee32fddbe60aadf24f14ce44 v1.13.1

From pypy.commits at gmail.com  Mon Oct 21 10:57:58 2019
From: pypy.commits at gmail.com (arigo)
Date: Mon, 21 Oct 2019 07:57:58 -0700 (PDT)
Subject: [pypy-commit] cffi default: hg merge release-1.13
Message-ID: <5dadc776.1c69fb81.bb337.0dba@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3307:3e7c1908678f
Date: 2019-10-21 16:57 +0200
http://bitbucket.org/cffi/cffi/changeset/3e7c1908678f/

Log:	hg merge release-1.13

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -22,3 +22,4 @@
 e0c76668ecf5ab1ad005799cfe1b7cb58610f155 v1.12.2
 bf80722dea36237710083315e336c81bc8571fd0 v1.12.3
 17c06e75f54cf5a31865961f0b498f99232a0bc8 v1.13.0
+9cc790d05bc5256eee32fddbe60aadf24f14ce44 v1.13.1
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -54,11 +54,11 @@
 
 * Checksums of the "source" package version 1.13.1:
 
-   - MD5: ...
+   - MD5: 824ad9f228fbc6ce203c334e2ff4ab8f
 
-   - SHA: ...
+   - SHA: 678dad3a3102ebcbeab8f5f132968afa444f6a54
 
-   - SHA256: ...
+   - SHA256: 558b3afef987cf4b17abd849e7bedf64ee12b28175d564d05b628a0f9355599b
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``

From pypy.commits at gmail.com  Mon Oct 21 17:49:22 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 21 Oct 2019 14:49:22 -0700 (PDT)
Subject: [pypy-commit] pypy default: reset extra_libs after creating the
 Makefile
Message-ID: <5dae27e2.1c69fb81.378f9.7259@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r97828:fb4d4c113dae
Date: 2019-10-21 22:47 +0300
http://bitbucket.org/pypy/pypy/changeset/fb4d4c113dae/

Log:	reset extra_libs after creating the Makefile

diff --git a/rpython/translator/platform/darwin.py b/rpython/translator/platform/darwin.py
--- a/rpython/translator/platform/darwin.py
+++ b/rpython/translator/platform/darwin.py
@@ -99,6 +99,7 @@
                      no_precompile_cfiles = [], profopt=False, config=None):
         # ensure frameworks are passed in the Makefile
         fs = self._frameworks(eci.frameworks)
+        extra_libs = self.extra_libs
         if len(fs) > 0:
             # concat (-framework, FrameworkName) pairs
             self.extra_libs += tuple(map(" ".join, zip(fs[::2], fs[1::2])))
@@ -107,6 +108,7 @@
                                 headers_to_precompile=headers_to_precompile,
                                 no_precompile_cfiles = no_precompile_cfiles,
                                 profopt=profopt, config=config)
+        self.extra_libs = extra_libs
         return mk
 
 class Darwin_PowerPC(Darwin):#xxx fixme, mwp

From pypy.commits at gmail.com  Mon Oct 21 17:49:26 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 21 Oct 2019 14:49:26 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: make the include of sys/time.h local to
 pytime.c, include it on macos
Message-ID: <5dae27e6.1c69fb81.42766.49fa@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r97830:806e2a942849
Date: 2019-10-22 00:45 +0300
http://bitbucket.org/pypy/pypy/changeset/806e2a942849/

Log:	make the include of sys/time.h local to pytime.c, include it on
	macos

diff --git a/pypy/module/cpyext/include/pyport.h b/pypy/module/cpyext/include/pyport.h
--- a/pypy/module/cpyext/include/pyport.h
+++ b/pypy/module/cpyext/include/pyport.h
@@ -41,9 +41,6 @@
 
 /* CPython needs this for the c-extension datetime, which is pure python on PyPy 
    downstream packages assume it is here (Pandas for instance) */
-#ifdef HAVE_CLOCK_GETTIME
-#include <sys/time.h>
-#endif
 #include <time.h> 
 
 /* uintptr_t is the C9X name for an unsigned integral type such that a
diff --git a/pypy/module/cpyext/src/pytime.c b/pypy/module/cpyext/src/pytime.c
--- a/pypy/module/cpyext/src/pytime.c
+++ b/pypy/module/cpyext/src/pytime.c
@@ -1,6 +1,8 @@
 #include "Python.h"
 #ifdef MS_WINDOWS
 #include <windows.h>
+#else
+#include <sys/time.h>
 #endif
 
 #if defined(__APPLE__)

From pypy.commits at gmail.com  Mon Oct 21 17:49:24 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 21 Oct 2019 14:49:24 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: merge default into branch
Message-ID: <5dae27e4.1c69fb81.62b46.044e@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r97829:5cd23d4955a1
Date: 2019-10-22 00:32 +0300
http://bitbucket.org/pypy/pypy/changeset/5cd23d4955a1/

Log:	merge default into branch

diff --git a/extra_tests/cffi_tests/cffi0/test_function.py b/extra_tests/cffi_tests/cffi0/test_function.py
--- a/extra_tests/cffi_tests/cffi0/test_function.py
+++ b/extra_tests/cffi_tests/cffi0/test_function.py
@@ -114,7 +114,7 @@
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
             int fputs(const char *, void *);
-            void *stderr;
+            extern void *stderr;
         """)
         needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
@@ -131,7 +131,7 @@
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
             int fputs(char *, void *);
-            void *stderr;
+            extern void *stderr;
         """)
         needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
@@ -148,7 +148,7 @@
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
            int fprintf(void *, const char *format, ...);
-           void *stderr;
+           extern void *stderr;
         """)
         needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
@@ -210,7 +210,7 @@
             py.test.skip("probably no symbol 'stderr' in the lib")
         ffi.cdef("""
             int fputs(const char *, void *);
-            void *stderr;
+            extern void *stderr;
         """)
         needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
@@ -257,7 +257,7 @@
             py.test.skip("probably no symbol 'stdout' in the lib")
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
-            void *stdout;
+            extern void *stdout;
         """)
         needs_dlopen_none()
         C = ffi.dlopen(None)
@@ -497,7 +497,7 @@
         ffi.cdef("""
             typedef enum { MYE1, MYE2 } myenum_t;
             double myfunc(double);
-            double myvar;
+            extern double myvar;
             const double myconst;
             #define MYFOO 42
         """)
@@ -508,7 +508,7 @@
         if self.Backend is CTypesBackend:
             py.test.skip("not with the ctypes backend")
         ffi = FFI(backend=self.Backend())
-        ffi.cdef("int foobar(void); int foobaz;")
+        ffi.cdef("int foobar(void); extern int foobaz;")
         lib = ffi.dlopen(lib_m)
         ffi.dlclose(lib)
         e = py.test.raises(ValueError, getattr, lib, 'foobar')
diff --git a/extra_tests/cffi_tests/cffi0/test_ownlib.py b/extra_tests/cffi_tests/cffi0/test_ownlib.py
--- a/extra_tests/cffi_tests/cffi0/test_ownlib.py
+++ b/extra_tests/cffi_tests/cffi0/test_ownlib.py
@@ -202,7 +202,7 @@
             py.test.skip("fix the auto-generation of the tiny test lib")
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
-            int my_array[7];
+            extern int my_array[7];
         """)
         ownlib = ffi.dlopen(self.module)
         for i in range(7):
@@ -224,7 +224,7 @@
             py.test.skip("not supported by the ctypes backend")
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
-            int my_array[];
+            extern int my_array[];
         """)
         ownlib = ffi.dlopen(self.module)
         for i in range(7):
@@ -292,7 +292,7 @@
                 long bottom;
             } RECT;
             
-            long left, top, right, bottom;
+            extern long left, top, right, bottom;
 
             RECT ReturnRect(int i, RECT ar, RECT* br, POINT cp, RECT dr,
                         RECT *er, POINT fp, RECT gr);
@@ -322,7 +322,7 @@
         if self.Backend is CTypesBackend:
             py.test.skip("not implemented with the ctypes backend")
         ffi = FFI(backend=self.Backend())
-        ffi.cdef("long left; int test_getting_errno(void);")
+        ffi.cdef("extern long left; int test_getting_errno(void);")
         lib = ffi.dlopen(self.module)
         lib.left = 123456
         p = ffi.addressof(lib, "left")
diff --git a/extra_tests/cffi_tests/cffi0/test_parsing.py b/extra_tests/cffi_tests/cffi0/test_parsing.py
--- a/extra_tests/cffi_tests/cffi0/test_parsing.py
+++ b/extra_tests/cffi_tests/cffi0/test_parsing.py
@@ -325,6 +325,7 @@
     assert value == sys.maxsize * 2 - 40
 
 def test__is_constant_globalvar():
+    import warnings
     for input, expected_output in [
         ("int a;",          False),
         ("const int a;",    True),
@@ -342,10 +343,13 @@
         ("const int a[5][6];", False),
         ]:
         ffi = FFI()
-        ffi.cdef(input)
+        with warnings.catch_warnings(record=True) as log:
+            warnings.simplefilter("always")
+            ffi.cdef(input)
         declarations = ffi._parser._declarations
         assert ('constant a' in declarations) == expected_output
         assert ('variable a' in declarations) == (not expected_output)
+        assert len(log) == (1 - expected_output)
 
 def test_restrict():
     from cffi import model
@@ -355,7 +359,7 @@
         ("int *a;",            False),
         ]:
         ffi = FFI()
-        ffi.cdef(input)
+        ffi.cdef("extern " + input)
         tp, quals = ffi._parser._declarations['variable a']
         assert bool(quals & model.Q_RESTRICT) == expected_output
 
diff --git a/extra_tests/cffi_tests/cffi0/test_verify.py b/extra_tests/cffi_tests/cffi0/test_verify.py
--- a/extra_tests/cffi_tests/cffi0/test_verify.py
+++ b/extra_tests/cffi_tests/cffi0/test_verify.py
@@ -287,7 +287,7 @@
 def test_var_signed_integer_types():
     ffi = FFI()
     lst = all_signed_integer_types(ffi)
-    csource = "\n".join(["%s somevar_%s;" % (tp, tp.replace(' ', '_'))
+    csource = "\n".join(["static %s somevar_%s;" % (tp, tp.replace(' ', '_'))
                          for tp in lst])
     ffi.cdef(csource)
     lib = ffi.verify(csource)
@@ -306,7 +306,7 @@
 def test_var_unsigned_integer_types():
     ffi = FFI()
     lst = all_unsigned_integer_types(ffi)
-    csource = "\n".join(["%s somevar_%s;" % (tp, tp.replace(' ', '_'))
+    csource = "\n".join(["static %s somevar_%s;" % (tp, tp.replace(' ', '_'))
                          for tp in lst])
     ffi.cdef(csource)
     lib = ffi.verify(csource)
@@ -818,8 +818,8 @@
 
 def test_access_variable():
     ffi = FFI()
-    ffi.cdef("int foo(void);\n"
-             "int somenumber;")
+    ffi.cdef("static int foo(void);\n"
+             "static int somenumber;")
     lib = ffi.verify("""
         static int somenumber = 2;
         static int foo(void) {
@@ -836,7 +836,7 @@
 def test_access_address_of_variable():
     # access the address of 'somenumber': need a trick
     ffi = FFI()
-    ffi.cdef("int somenumber; static int *const somenumberptr;")
+    ffi.cdef("static int somenumber; static int *const somenumberptr;")
     lib = ffi.verify("""
         static int somenumber = 2;
         #define somenumberptr (&somenumber)
@@ -849,7 +849,7 @@
 def test_access_array_variable(length=5):
     ffi = FFI()
     ffi.cdef("int foo(int);\n"
-             "int somenumber[%s];" % (length,))
+             "static int somenumber[%s];" % (length,))
     lib = ffi.verify("""
         static int somenumber[] = {2, 2, 3, 4, 5};
         static int foo(int i) {
@@ -881,7 +881,7 @@
     ffi = FFI()
     ffi.cdef("struct foo { int x; ...; };\n"
              "int foo(int);\n"
-             "struct foo stuff;")
+             "static struct foo stuff;")
     lib = ffi.verify("""
         struct foo { int x, y, z; };
         static struct foo stuff = {2, 5, 8};
@@ -905,9 +905,9 @@
 
 def test_access_callback():
     ffi = FFI()
-    ffi.cdef("int (*cb)(int);\n"
-             "int foo(int);\n"
-             "void reset_cb(void);")
+    ffi.cdef("static int (*cb)(int);\n"
+             "static int foo(int);\n"
+             "static void reset_cb(void);")
     lib = ffi.verify("""
         static int g(int x) { return x * 7; }
         static int (*cb)(int);
@@ -923,9 +923,9 @@
 def test_access_callback_function_typedef():
     ffi = FFI()
     ffi.cdef("typedef int mycallback_t(int);\n"
-             "mycallback_t *cb;\n"
-             "int foo(int);\n"
-             "void reset_cb(void);")
+             "static mycallback_t *cb;\n"
+             "static int foo(int);\n"
+             "static void reset_cb(void);")
     lib = ffi.verify("""
         static int g(int x) { return x * 7; }
         static int (*cb)(int);
@@ -1075,7 +1075,7 @@
 def test_autofilled_struct_as_argument_dynamic():
     ffi = FFI()
     ffi.cdef("struct foo_s { long a; ...; };\n"
-             "int (*foo)(struct foo_s);")
+             "static int (*foo)(struct foo_s);")
     lib = ffi.verify("""
         struct foo_s {
             double b;
@@ -1084,7 +1084,7 @@
         int foo1(struct foo_s s) {
             return (int)s.a - (int)s.b;
         }
-        int (*foo)(struct foo_s s) = &foo1;
+        static int (*foo)(struct foo_s s) = &foo1;
     """)
     e = py.test.raises(NotImplementedError, lib.foo, "?")
     msg = ("ctype 'struct foo_s' not supported as argument.  It is a struct "
@@ -1454,7 +1454,7 @@
         py.test.skip("_Bool not in MSVC")
     ffi = FFI()
     ffi.cdef("struct foo_s { _Bool x; };"
-             "_Bool foo(_Bool); _Bool (*foop)(_Bool);")
+             "_Bool foo(_Bool); static _Bool (*foop)(_Bool);")
     lib = ffi.verify("""
         struct foo_s { _Bool x; };
         int foo(int arg) {
@@ -1463,7 +1463,7 @@
         _Bool _foofunc(_Bool x) {
             return !x;
         }
-        _Bool (*foop)(_Bool) = _foofunc;
+        static _Bool (*foop)(_Bool) = _foofunc;
     """)
     p = ffi.new("struct foo_s *")
     p.x = 1
@@ -1654,7 +1654,7 @@
 
 def test_FILE_stored_explicitly():
     ffi = FFI()
-    ffi.cdef("int myprintf11(const char *, int); FILE *myfile;")
+    ffi.cdef("int myprintf11(const char *, int); extern FILE *myfile;")
     lib = ffi.verify("""
         #include <stdio.h>
         FILE *myfile;
@@ -1680,19 +1680,19 @@
 
 def test_global_array_with_missing_length():
     ffi = FFI()
-    ffi.cdef("int fooarray[];")
+    ffi.cdef("extern int fooarray[];")
     lib = ffi.verify("int fooarray[50];")
     assert repr(lib.fooarray).startswith("<cdata 'int *'")
 
 def test_global_array_with_dotdotdot_length():
     ffi = FFI()
-    ffi.cdef("int fooarray[...];")
+    ffi.cdef("extern int fooarray[...];")
     lib = ffi.verify("int fooarray[50];")
     assert repr(lib.fooarray).startswith("<cdata 'int[50]'")
 
 def test_bad_global_array_with_dotdotdot_length():
     ffi = FFI()
-    ffi.cdef("int fooarray[...];")
+    ffi.cdef("extern int fooarray[...];")
     py.test.raises(VerificationError, ffi.verify, "char fooarray[23];")
 
 def test_struct_containing_struct():
@@ -1813,7 +1813,7 @@
 def test_callback_indirection():
     ffi = FFI()
     ffi.cdef("""
-        int (*python_callback)(int how_many, int *values);
+        static int (*python_callback)(int how_many, int *values);
         int (*const c_callback)(int,...);   /* pass this ptr to C routines */
         int some_c_function(int(*cb)(int,...));
     """)
@@ -1947,24 +1947,24 @@
 
 def test_bug_const_char_ptr_array_1():
     ffi = FFI()
-    ffi.cdef("""const char *a[...];""")
+    ffi.cdef("""extern const char *a[...];""")
     lib = ffi.verify("""const char *a[5];""")
     assert repr(ffi.typeof(lib.a)) == "<ctype 'char *[5]'>"
 
 def test_bug_const_char_ptr_array_2():
     from cffi import FFI     # ignore warnings
     ffi = FFI()
-    ffi.cdef("""const int a[];""")
+    ffi.cdef("""extern const int a[];""")
     lib = ffi.verify("""const int a[5];""")
     assert repr(ffi.typeof(lib.a)) == "<ctype 'int *'>"
 
 def _test_various_calls(force_libffi):
     cdef_source = """
-    int xvalue;
-    long long ivalue, rvalue;
-    float fvalue;
-    double dvalue;
-    long double Dvalue;
+    extern int xvalue;
+    extern long long ivalue, rvalue;
+    extern float fvalue;
+    extern double dvalue;
+    extern long double Dvalue;
     signed char tf_bb(signed char x, signed char c);
     unsigned char tf_bB(signed char x, unsigned char c);
     short tf_bh(signed char x, short c);
@@ -2148,7 +2148,7 @@
     # exported symbols as well.  So we must not export a simple name
     # like 'foo'!
     ffi1 = FFI()
-    ffi1.cdef("int foo_verify_dlopen_flags;")
+    ffi1.cdef("extern int foo_verify_dlopen_flags;")
 
     lib1 = ffi1.verify("int foo_verify_dlopen_flags;",
                        flags=ffi1.RTLD_GLOBAL | ffi1.RTLD_LAZY)
@@ -2162,7 +2162,7 @@
 def get_second_lib():
     # Hack, using modulename makes the test fail
     ffi2 = FFI()
-    ffi2.cdef("int foo_verify_dlopen_flags;")
+    ffi2.cdef("extern int foo_verify_dlopen_flags;")
     lib2 = ffi2.verify("int foo_verify_dlopen_flags;",
                        flags=ffi2.RTLD_GLOBAL | ffi2.RTLD_LAZY)
     return lib2
diff --git a/extra_tests/cffi_tests/cffi1/test_dlopen.py b/extra_tests/cffi_tests/cffi1/test_dlopen.py
--- a/extra_tests/cffi_tests/cffi1/test_dlopen.py
+++ b/extra_tests/cffi_tests/cffi1/test_dlopen.py
@@ -7,7 +7,7 @@
 
 def test_simple():
     ffi = FFI()
-    ffi.cdef("int close(int); static const int BB = 42; int somevar;")
+    ffi.cdef("int close(int); static const int BB = 42; extern int somevar;")
     target = udir.join('test_simple.py')
     make_py_source(ffi, 'test_simple', str(target))
     assert target.read() == r"""# auto-generated file
@@ -197,7 +197,7 @@
 
 def test_global_var():
     ffi = FFI()
-    ffi.cdef("int myglob;")
+    ffi.cdef("extern int myglob;")
     target = udir.join('test_global_var.py')
     make_py_source(ffi, 'test_global_var', str(target))
     assert target.read() == r"""# auto-generated file
diff --git a/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py
--- a/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py
+++ b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py
@@ -1780,7 +1780,7 @@
 
     def test_import_from_lib(self):
         ffi2 = cffi.FFI()
-        ffi2.cdef("int myfunc(int); int myvar;\n#define MYFOO ...\n")
+        ffi2.cdef("int myfunc(int); extern int myvar;\n#define MYFOO ...\n")
         outputfilename = recompile(ffi2, "_test_import_from_lib",
                                    "int myfunc(int x) { return x + 1; }\n"
                                    "int myvar = -5;\n"
diff --git a/extra_tests/cffi_tests/cffi1/test_re_python.py b/extra_tests/cffi_tests/cffi1/test_re_python.py
--- a/extra_tests/cffi_tests/cffi1/test_re_python.py
+++ b/extra_tests/cffi_tests/cffi1/test_re_python.py
@@ -64,11 +64,11 @@
     #define BIGNEG -420000000000L
     int add42(int);
     int add43(int, ...);
-    int globalvar42;
+    extern int globalvar42;
     const int globalconst42;
     const char *const globalconsthello;
     int no_such_function(int);
-    int no_such_globalvar;
+    extern int no_such_globalvar;
     struct foo_s;
     typedef struct bar_s { int x; signed char a[]; } bar_t;
     enum foo_e { AA, BB, CC };
@@ -76,6 +76,7 @@
     struct with_union { union { int a; char b; }; };
     union with_struct { struct { int a; char b; }; };
     struct NVGcolor { union { float rgba[4]; struct { float r,g,b,a; }; }; };
+    typedef struct selfref { struct selfref *next; } *selfref_ptr_t;
     """)
     ffi.set_source('re_python_pysrc', None)
     ffi.emit_python_code(str(tmpdir.join('re_python_pysrc.py')))
@@ -255,3 +256,8 @@
     assert ffi.offsetof("struct NVGcolor", "g") == FLOAT
     assert ffi.offsetof("struct NVGcolor", "b") == FLOAT * 2
     assert ffi.offsetof("struct NVGcolor", "a") == FLOAT * 3
+
+def test_selfref():
+    # based on issue #429
+    from re_python_pysrc import ffi
+    ffi.new("selfref_ptr_t")
diff --git a/extra_tests/cffi_tests/cffi1/test_recompiler.py b/extra_tests/cffi_tests/cffi1/test_recompiler.py
--- a/extra_tests/cffi_tests/cffi1/test_recompiler.py
+++ b/extra_tests/cffi_tests/cffi1/test_recompiler.py
@@ -84,7 +84,7 @@
                      "(FUNCTION 1)(PRIMITIVE 7)(FUNCTION_END 1)(POINTER 0)")
 
 def test_type_table_array():
-    check_type_table("int a[100];",
+    check_type_table("extern int a[100];",
                      "(PRIMITIVE 7)(ARRAY 0)(None 100)")
 
 def test_type_table_typedef():
@@ -159,7 +159,7 @@
 
 def test_global_var_array():
     ffi = FFI()
-    ffi.cdef("int a[100];")
+    ffi.cdef("extern int a[100];")
     lib = verify(ffi, 'test_global_var_array', 'int a[100] = { 9999 };')
     lib.a[42] = 123456
     assert lib.a[42] == 123456
@@ -183,7 +183,7 @@
 
 def test_global_var_int():
     ffi = FFI()
-    ffi.cdef("int a, b, c;")
+    ffi.cdef("extern int a, b, c;")
     lib = verify(ffi, 'test_global_var_int', 'int a = 999, b, c;')
     assert lib.a == 999
     lib.a -= 1001
@@ -284,7 +284,7 @@
 
 def test_dir():
     ffi = FFI()
-    ffi.cdef("int ff(int); int aa; static const int my_constant;")
+    ffi.cdef("int ff(int); extern int aa; static const int my_constant;")
     lib = verify(ffi, 'test_dir', """
         #define my_constant  (-45)
         int aa;
@@ -406,7 +406,7 @@
 
 def test_dotdotdot_global_array():
     ffi = FFI()
-    ffi.cdef("int aa[...]; int bb[...];")
+    ffi.cdef("extern int aa[...]; extern int bb[...];")
     lib = verify(ffi, 'test_dotdotdot_global_array',
                  "int aa[41]; int bb[12];")
     assert ffi.sizeof(lib.aa) == 41 * 4
@@ -561,37 +561,37 @@
 
 def test_bad_size_of_global_1():
     ffi = FFI()
-    ffi.cdef("short glob;")
+    ffi.cdef("extern short glob;")
     py.test.raises(VerificationError, verify, ffi,
                    "test_bad_size_of_global_1", "long glob;")
 
 def test_bad_size_of_global_2():
     ffi = FFI()
-    ffi.cdef("int glob[10];")
+    ffi.cdef("extern int glob[10];")
     py.test.raises(VerificationError, verify, ffi,
                    "test_bad_size_of_global_2", "int glob[9];")
 
 def test_unspecified_size_of_global_1():
     ffi = FFI()
-    ffi.cdef("int glob[];")
+    ffi.cdef("extern int glob[];")
     lib = verify(ffi, "test_unspecified_size_of_global_1", "int glob[10];")
     assert ffi.typeof(lib.glob) == ffi.typeof("int *")
 
 def test_unspecified_size_of_global_2():
     ffi = FFI()
-    ffi.cdef("int glob[][5];")
+    ffi.cdef("extern int glob[][5];")
     lib = verify(ffi, "test_unspecified_size_of_global_2", "int glob[10][5];")
     assert ffi.typeof(lib.glob) == ffi.typeof("int(*)[5]")
 
 def test_unspecified_size_of_global_3():
     ffi = FFI()
-    ffi.cdef("int glob[][...];")
+    ffi.cdef("extern int glob[][...];")
     lib = verify(ffi, "test_unspecified_size_of_global_3", "int glob[10][5];")
     assert ffi.typeof(lib.glob) == ffi.typeof("int(*)[5]")
 
 def test_unspecified_size_of_global_4():
     ffi = FFI()
-    ffi.cdef("int glob[...][...];")
+    ffi.cdef("extern int glob[...][...];")
     lib = verify(ffi, "test_unspecified_size_of_global_4", "int glob[10][5];")
     assert ffi.typeof(lib.glob) == ffi.typeof("int[10][5]")
 
@@ -814,7 +814,7 @@
 def test_address_of_global_var():
     ffi = FFI()
     ffi.cdef("""
-        long bottom, bottoms[2];
+        extern long bottom, bottoms[2];
         long FetchRectBottom(void);
         long FetchRectBottoms1(void);
         #define FOOBAR 42
@@ -969,7 +969,7 @@
     ffi = FFI()
     ffi.cdef("""
         typedef ... opaque_t;
-        opaque_t globvar;
+        extern opaque_t globvar;
     """)
     lib = verify(ffi, 'test_variable_of_unknown_size', """
         typedef char opaque_t[6];
@@ -1014,7 +1014,7 @@
 def test_call_with_incomplete_structs():
     ffi = FFI()
     ffi.cdef("typedef struct {...;} foo_t; "
-             "foo_t myglob; "
+             "extern foo_t myglob; "
              "foo_t increment(foo_t s); "
              "double getx(foo_t s);")
     lib = verify(ffi, 'test_call_with_incomplete_structs', """
@@ -1058,7 +1058,7 @@
 
 def test_global_var_array_2():
     ffi = FFI()
-    ffi.cdef("int a[...][...];")
+    ffi.cdef("extern int a[...][...];")
     lib = verify(ffi, 'test_global_var_array_2', 'int a[10][8];')
     lib.a[9][7] = 123456
     assert lib.a[9][7] == 123456
@@ -1071,7 +1071,7 @@
 
 def test_global_var_array_3():
     ffi = FFI()
-    ffi.cdef("int a[][...];")
+    ffi.cdef("extern int a[][...];")
     lib = verify(ffi, 'test_global_var_array_3', 'int a[10][8];')
     lib.a[9][7] = 123456
     assert lib.a[9][7] == 123456
@@ -1082,7 +1082,7 @@
 
 def test_global_var_array_4():
     ffi = FFI()
-    ffi.cdef("int a[10][...];")
+    ffi.cdef("extern int a[10][...];")
     lib = verify(ffi, 'test_global_var_array_4', 'int a[10][8];')
     lib.a[9][7] = 123456
     assert lib.a[9][7] == 123456
@@ -1205,7 +1205,7 @@
 
 def test_import_from_lib():
     ffi = FFI()
-    ffi.cdef("int mybar(int); int myvar;\n#define MYFOO ...")
+    ffi.cdef("int mybar(int); static int myvar;\n#define MYFOO ...")
     lib = verify(ffi, 'test_import_from_lib',
                  "#define MYFOO 42\n"
                  "static int mybar(int x) { return x + 1; }\n"
@@ -1221,7 +1221,7 @@
 
 def test_macro_var_callback():
     ffi = FFI()
-    ffi.cdef("int my_value; int *(*get_my_value)(void);")
+    ffi.cdef("extern int my_value; extern int *(*get_my_value)(void);")
     lib = verify(ffi, 'test_macro_var_callback',
                  "int *(*get_my_value)(void);\n"
                  "#define my_value (*get_my_value())")
@@ -1336,7 +1336,7 @@
 
 def test_const_function_type_args():
     ffi = FFI()
-    ffi.cdef("""int (*foobar)(const int a, const int *b, const int c[]);""")
+    ffi.cdef("""extern int(*foobar)(const int a,const int*b,const int c[]);""")
     lib = verify(ffi, 'test_const_function_type_args', """
         int (*foobar)(const int a, const int *b, const int c[]);
     """)
@@ -1626,7 +1626,7 @@
 
 def test_extern_python_bogus_name():
     ffi = FFI()
-    ffi.cdef("int abc;")
+    ffi.cdef("extern int abc;")
     lib = verify(ffi, 'test_extern_python_bogus_name', "int abc;")
     def fn():
         pass
@@ -1787,8 +1787,8 @@
     ffi.cdef("""
         extern "Python" int __stdcall foo(int);
         extern "Python" int WINAPI bar(int);
-        int (__stdcall * mycb1)(int);
-        int indirect_call(int);
+        static int (__stdcall * mycb1)(int);
+        static int indirect_call(int);
     """)
     lib = verify(ffi, 'test_extern_python_stdcall', """
         #ifndef _MSC_VER
@@ -1856,7 +1856,7 @@
 
 def test_introspect_global_var():
     ffi = FFI()
-    ffi.cdef("float g1;")
+    ffi.cdef("extern float g1;")
     lib = verify(ffi, 'test_introspect_global_var', """
         float g1;
     """)
@@ -1867,7 +1867,7 @@
 
 def test_introspect_global_var_array():
     ffi = FFI()
-    ffi.cdef("float g1[100];")
+    ffi.cdef("extern float g1[100];")
     lib = verify(ffi, 'test_introspect_global_var_array', """
         float g1[100];
     """)
@@ -2090,7 +2090,7 @@
     ffi = FFI()
     ffi.cdef("""
         typedef int foo_t[...], bar_t[...];
-        int gv[...];
+        extern int gv[...];
         typedef int mat_t[...][...];
         typedef int vmat_t[][...];
         """)
diff --git a/extra_tests/cffi_tests/cffi1/test_verify1.py b/extra_tests/cffi_tests/cffi1/test_verify1.py
--- a/extra_tests/cffi_tests/cffi1/test_verify1.py
+++ b/extra_tests/cffi_tests/cffi1/test_verify1.py
@@ -268,7 +268,7 @@
 def test_var_signed_integer_types():
     ffi = FFI()
     lst = all_signed_integer_types(ffi)
-    csource = "\n".join(["%s somevar_%s;" % (tp, tp.replace(' ', '_'))
+    csource = "\n".join(["static %s somevar_%s;" % (tp, tp.replace(' ', '_'))
                          for tp in lst])
     ffi.cdef(csource)
     lib = ffi.verify(csource)
@@ -287,7 +287,7 @@
 def test_var_unsigned_integer_types():
     ffi = FFI()
     lst = all_unsigned_integer_types(ffi)
-    csource = "\n".join(["%s somevar_%s;" % (tp, tp.replace(' ', '_'))
+    csource = "\n".join(["static %s somevar_%s;" % (tp, tp.replace(' ', '_'))
                          for tp in lst])
     ffi.cdef(csource)
     lib = ffi.verify(csource)
@@ -791,8 +791,8 @@
 
 def test_access_variable():
     ffi = FFI()
-    ffi.cdef("int foo(void);\n"
-             "int somenumber;")
+    ffi.cdef("static int foo(void);\n"
+             "static int somenumber;")
     lib = ffi.verify("""
         static int somenumber = 2;
         static int foo(void) {
@@ -809,7 +809,7 @@
 def test_access_address_of_variable():
     # access the address of 'somenumber': need a trick
     ffi = FFI()
-    ffi.cdef("int somenumber; static int *const somenumberptr;")
+    ffi.cdef("static int somenumber; static int *const somenumberptr;")
     lib = ffi.verify("""
         static int somenumber = 2;
         #define somenumberptr (&somenumber)
@@ -821,8 +821,8 @@
 
 def test_access_array_variable(length=5):
     ffi = FFI()
-    ffi.cdef("int foo(int);\n"
-             "int somenumber[%s];" % (length,))
+    ffi.cdef("static int foo(int);\n"
+             "static int somenumber[%s];" % (length,))
     lib = ffi.verify("""
         static int somenumber[] = {2, 2, 3, 4, 5};
         static int foo(int i) {
@@ -853,8 +853,8 @@
 def test_access_struct_variable():
     ffi = FFI()
     ffi.cdef("struct foo { int x; ...; };\n"
-             "int foo(int);\n"
-             "struct foo stuff;")
+             "static int foo(int);\n"
+             "static struct foo stuff;")
     lib = ffi.verify("""
         struct foo { int x, y, z; };
         static struct foo stuff = {2, 5, 8};
@@ -878,9 +878,9 @@
 
 def test_access_callback():
     ffi = FFI()
-    ffi.cdef("int (*cb)(int);\n"
-             "int foo(int);\n"
-             "void reset_cb(void);")
+    ffi.cdef("static int (*cb)(int);\n"
+             "static int foo(int);\n"
+             "static void reset_cb(void);")
     lib = ffi.verify("""
         static int g(int x) { return x * 7; }
         static int (*cb)(int);
@@ -896,9 +896,9 @@
 def test_access_callback_function_typedef():
     ffi = FFI()
     ffi.cdef("typedef int mycallback_t(int);\n"
-             "mycallback_t *cb;\n"
-             "int foo(int);\n"
-             "void reset_cb(void);")
+             "static mycallback_t *cb;\n"
+             "static int foo(int);\n"
+             "static void reset_cb(void);")
     lib = ffi.verify("""
         static int g(int x) { return x * 7; }
         static int (*cb)(int);
@@ -1039,7 +1039,7 @@
 def test_autofilled_struct_as_argument_dynamic():
     ffi = FFI()
     ffi.cdef("struct foo_s { long a; ...; };\n"
-             "int (*foo)(struct foo_s);")
+             "static int (*foo)(struct foo_s);")
     lib = ffi.verify("""
         struct foo_s {
             double b;
@@ -1048,7 +1048,7 @@
         int foo1(struct foo_s s) {
             return (int)s.a - (int)s.b;
         }
-        int (*foo)(struct foo_s s) = &foo1;
+        static int (*foo)(struct foo_s s) = &foo1;
     """)
     e = py.test.raises(NotImplementedError, lib.foo, "?")
     msg = ("ctype 'struct foo_s' not supported as argument.  It is a struct "
@@ -1424,7 +1424,7 @@
         py.test.skip("_Bool not in MSVC")
     ffi = FFI()
     ffi.cdef("struct foo_s { _Bool x; };"
-             "_Bool foo(_Bool); _Bool (*foop)(_Bool);")
+             "_Bool foo(_Bool); static _Bool (*foop)(_Bool);")
     lib = ffi.verify("""
         struct foo_s { _Bool x; };
         int foo(int arg) {
@@ -1433,7 +1433,7 @@
         _Bool _foofunc(_Bool x) {
             return !x;
         }
-        _Bool (*foop)(_Bool) = _foofunc;
+        static _Bool (*foop)(_Bool) = _foofunc;
     """)
     p = ffi.new("struct foo_s *")
     p.x = 1
@@ -1618,7 +1618,7 @@
 
 def test_FILE_stored_explicitly():
     ffi = FFI()
-    ffi.cdef("int myprintf11(const char *, int); FILE *myfile;")
+    ffi.cdef("int myprintf11(const char *, int); extern FILE *myfile;")
     lib = ffi.verify("""
         #include <stdio.h>
         FILE *myfile;
@@ -1644,13 +1644,13 @@
 
 def test_global_array_with_missing_length():
     ffi = FFI()
-    ffi.cdef("int fooarray[];")
+    ffi.cdef("extern int fooarray[];")
     lib = ffi.verify("int fooarray[50];")
     assert repr(lib.fooarray).startswith("<cdata 'int *'")
 
 def test_global_array_with_dotdotdot_length():
     ffi = FFI()
-    ffi.cdef("int fooarray[...];")
+    ffi.cdef("extern int fooarray[...];")
     lib = ffi.verify("int fooarray[50];")
     assert repr(lib.fooarray).startswith("<cdata 'int[50]'")
 
@@ -1658,7 +1658,7 @@
     py.test.xfail("was detected only because 23 bytes cannot be divided by 4; "
                   "redo more generally")
     ffi = FFI()
-    ffi.cdef("int fooarray[...];")
+    ffi.cdef("extern int fooarray[...];")
     py.test.raises(VerificationError, ffi.verify, "char fooarray[23];")
 
 def test_struct_containing_struct():
@@ -1779,7 +1779,7 @@
 def test_callback_indirection():
     ffi = FFI()
     ffi.cdef("""
-        int (*python_callback)(int how_many, int *values);
+        static int (*python_callback)(int how_many, int *values);
         int (*const c_callback)(int,...);   /* pass this ptr to C routines */
         int some_c_function(int(*cb)(int,...));
     """)
@@ -1913,23 +1913,23 @@
 
 def test_bug_const_char_ptr_array_1():
     ffi = FFI()
-    ffi.cdef("""const char *a[...];""")
+    ffi.cdef("""extern const char *a[...];""")
     lib = ffi.verify("""const char *a[5];""")
     assert repr(ffi.typeof(lib.a)) == "<ctype 'char *[5]'>"
 
 def test_bug_const_char_ptr_array_2():
     ffi = FFI()
-    ffi.cdef("""const int a[];""")
+    ffi.cdef("""extern const int a[];""")
     lib = ffi.verify("""const int a[5];""")
     assert repr(ffi.typeof(lib.a)) == "<ctype 'int *'>"
 
 def _test_various_calls(force_libffi):
     cdef_source = """
-    int xvalue;
-    long long ivalue, rvalue;
-    float fvalue;
-    double dvalue;
-    long double Dvalue;
+    extern int xvalue;
+    extern long long ivalue, rvalue;
+    extern float fvalue;
+    extern double dvalue;
+    extern long double Dvalue;
     signed char tf_bb(signed char x, signed char c);
     unsigned char tf_bB(signed char x, unsigned char c);
     short tf_bh(signed char x, short c);
@@ -2112,7 +2112,7 @@
     old = sys.getdlopenflags()
     try:
         ffi1 = FFI()
-        ffi1.cdef("int foo_verify_dlopen_flags_1;")
+        ffi1.cdef("extern int foo_verify_dlopen_flags_1;")
         sys.setdlopenflags(ffi1.RTLD_GLOBAL | ffi1.RTLD_NOW)
         lib1 = ffi1.verify("int foo_verify_dlopen_flags_1;")
     finally:
@@ -2253,7 +2253,7 @@
 
 def test_macro_var():
     ffi = FFI()
-    ffi.cdef("int myarray[50], my_value;")
+    ffi.cdef("extern int myarray[50], my_value;")
     lib = ffi.verify("""
         int myarray[50];
         int *get_my_value(void) {
diff --git a/extra_tests/cffi_tests/embedding/add1.py b/extra_tests/cffi_tests/embedding/add1.py
--- a/extra_tests/cffi_tests/embedding/add1.py
+++ b/extra_tests/cffi_tests/embedding/add1.py
@@ -12,7 +12,11 @@
     sys.stdout.write("preparing")
     for i in range(3):
         sys.stdout.flush()
-        time.sleep(0.2)
+        # Windows: sometimes time.sleep() doesn't sleep at all.
+        # This appears to occur on recent versions of python only.
+        t_end = time.time() + 0.19
+        while time.time() < t_end:
+            time.sleep(0.2)
         sys.stdout.write(".")
     sys.stdout.write("\n")
 
diff --git a/extra_tests/cffi_tests/embedding/add_recursive.py b/extra_tests/cffi_tests/embedding/add_recursive.py
--- a/extra_tests/cffi_tests/embedding/add_recursive.py
+++ b/extra_tests/cffi_tests/embedding/add_recursive.py
@@ -4,7 +4,7 @@
 ffi = cffi.FFI()
 
 ffi.embedding_api("""
-    int (*my_callback)(int);
+    extern int (*my_callback)(int);
     int add_rec(int, int);
 """)
 
diff --git a/extra_tests/cffi_tests/embedding/test_thread.py b/extra_tests/cffi_tests/embedding/test_thread.py
--- a/extra_tests/cffi_tests/embedding/test_thread.py
+++ b/extra_tests/cffi_tests/embedding/test_thread.py
@@ -22,17 +22,21 @@
         add1_cffi = self.prepare_module('add1')
         add2_cffi = self.prepare_module('add2')
         self.compile('thread2-test', [add1_cffi, add2_cffi], threads=True)
-        output = self.execute('thread2-test')
-        output = self._take_out(output, "preparing")
-        output = self._take_out(output, ".")
-        output = self._take_out(output, ".")
-        # at least the 3rd dot should be after everything from ADD2
-        assert output == ("starting\n"
-                          "prepADD2\n"
-                          "adding 1000 and 200 and 30\n"
-                          ".\n"
-                          "adding 40 and 2\n"
-                          "done\n")
+        for i in range(3):
+            output = self.execute('thread2-test')
+            print('='*79)
+            print(output)
+            print('='*79)
+            output = self._take_out(output, "preparing")
+            output = self._take_out(output, ".")
+            output = self._take_out(output, ".")
+            # at least the 3rd dot should be after everything from ADD2
+            assert output == ("starting\n"
+                              "prepADD2\n"
+                              "adding 1000 and 200 and 30\n"
+                              ".\n"
+                              "adding 40 and 2\n"
+                              "done\n")
 
     def test_alt_issue(self):
         add1_cffi = self.prepare_module('add1')
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.13.0
+Version: 1.13.1
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -5,8 +5,8 @@
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 from .error import PkgConfigError
 
-__version__ = "1.13.0"
-__version_info__ = (1, 13, 0)
+__version__ = "1.13.1"
+__version_info__ = (1, 13, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -224,7 +224,7 @@
 
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.13.0"
+                               "\ncompiled with cffi version: 1.13.1"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
@@ -327,13 +327,15 @@
 #endif
 
     /* call Py_InitializeEx() */
-    {
-        PyGILState_STATE state = PyGILState_UNLOCKED;
-        if (!Py_IsInitialized())
-            _cffi_py_initialize();
-        else
-            state = PyGILState_Ensure();
-
+    if (!Py_IsInitialized()) {
+        _cffi_py_initialize();
+        PyEval_InitThreads();
+        PyEval_SaveThread();  /* release the GIL */
+        /* the returned tstate must be the one that has been stored into the
+           autoTLSkey by _PyGILState_Init() called from Py_Initialize(). */
+    }
+    else {
+        PyGILState_STATE state = PyGILState_Ensure();
         PyEval_InitThreads();
         PyGILState_Release(state);
     }
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -156,6 +156,13 @@
                           "confuse pre-parsing.")
             break
 
+def _warn_for_non_extern_non_static_global_variable(decl):
+    if not decl.storage:
+        import warnings
+        warnings.warn("Declaration of global variable '%s' in cdef() should "
+                      "be marked 'extern' for consistency (or possibly "
+                      "'static' in API mode)" % (decl.name,))
+
 def _preprocess(csource):
     # Remove comments.  NOTE: this only work because the cdef() section
     # should not contain any string literal!
@@ -506,6 +513,7 @@
                     if (quals & model.Q_CONST) and not tp.is_array_type:
                         self._declare('constant ' + decl.name, tp, quals=quals)
                     else:
+                        _warn_for_non_extern_non_static_global_variable(decl)
                         self._declare('variable ' + decl.name, tp, quals=quals)
 
     def parse_type(self, cdecl):
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -43,6 +43,7 @@
 
 .. toctree::
     whatsnew-pypy3-head.rst
+    whatsnew-pypy3-7.2.0.rst
     whatsnew-pypy3-7.1.0.rst
 
 CPython 3.5 compatible versions
diff --git a/pypy/doc/whatsnew-pypy3-5.10.0.rst b/pypy/doc/whatsnew-pypy3-5.10.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy3-5.10.0.rst
@@ -0,0 +1,7 @@
+========================
+What's new in PyPy3 7.0+
+========================
+
+.. this is the revision after release-pypy3.5-v7.0
+.. startrev: 9d2fa7c63b7c
+
diff --git a/pypy/doc/whatsnew-pypy3-6.0.0.rst b/pypy/doc/whatsnew-pypy3-6.0.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy3-6.0.0.rst
@@ -0,0 +1,7 @@
+========================
+What's new in PyPy3 7.0+
+========================
+
+.. this is the revision after release-pypy3.5-v7.0
+.. startrev: 9d2fa7c63b7c
+
diff --git a/pypy/doc/whatsnew-pypy3-7.1.0.rst b/pypy/doc/whatsnew-pypy3-7.1.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy3-7.1.0.rst
@@ -0,0 +1,11 @@
+========================
+What's new in PyPy3 7.0+
+========================
+
+.. this is the revision after release-pypy3.6-v7.0
+.. startrev: 33fe3b2cf186
+
+.. branch: py3.5
+
+Merge in py.35 and use this branch as the primary pypy3 one
+
diff --git a/pypy/doc/whatsnew-pypy3-7.2.0.rst b/pypy/doc/whatsnew-pypy3-7.2.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy3-7.2.0.rst
@@ -0,0 +1,58 @@
+=========================
+What's new in PyPy3 7.2.0
+=========================
+
+.. this is the revision after release-pypy3.6-v7.1.1
+.. startrev: db5a1e7fbbd0
+
+.. branch: fix-literal-prev_digit-underscore
+
+Fix parsing for converting strings with underscore into ints
+
+.. branch: winmultiprocessing
+
+Improve multiprocessing support on win32
+
+.. branch: setitem2d
+
+Allow 2d indexing in ``memoryview.__setitem__`` (issue bb-3028)
+
+.. branch: py3.6-socket-fix
+.. branch: fix-importerror
+.. branch: dj_s390
+.. branch: bpo-35409
+.. branch: remove_array_with_char_test
+.. branch: fix_test_unicode_outofrange
+.. branch: Ram-Rachum/faulthandleris_enabled-should-return-fal-1563636614875
+.. branch: Anthony-Sottile/fix-leak-of-file-descriptor-with-_iofile-1559687440863
+
+.. branch: py3tests
+
+Add handling of application-level test files and -D flag to test runner
+
+.. branch: vendor/stdlib-3.6
+.. branch: stdlib-3.6.9
+
+Update standard library to version 3.6.9
+
+.. branch: __debug__-optimize
+
+Fix handling of __debug__, sys.flags.optimize, and '-O' command-line flag to 
+match CPython 3.6.
+
+.. branch: more-cpyext
+
+Add ``PyErr_SetFromWindowsErr`` and ``pytime.h``, ``pytime.c``. Fix order of
+fields in ``Py_buffer``.
+
+.. branch: Ryan-Hileman/add-support-for-zipfile-stdlib-1562420744699
+
+Add support for the entire stdlib being inside a zipfile
+
+
+.. branch: json-decoder-maps-py3.6
+
+Much faster and more memory-efficient JSON decoding. The resulting
+dictionaries that come out of the JSON decoder have faster lookups too.
+
+
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -1164,7 +1164,7 @@
         # issue bb-3091 crash in BuiltinCodePassThroughArguments0.funcrun
         import sys
         if '__pypy__' in sys.modules:
-            msg_fmt = "'%s' object expected, got '%s'"
+            msg_fmt = "%s instance as first argument (got %s"
         else:
             msg_fmt = "'%s' object but received a '%s'"
         for obj in (dict, set):
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -1,1 +1,1 @@
-VERSION = "1.13.0"
+VERSION = "1.13.1"
diff --git a/pypy/module/_cffi_backend/realize_c_type.py b/pypy/module/_cffi_backend/realize_c_type.py
--- a/pypy/module/_cffi_backend/realize_c_type.py
+++ b/pypy/module/_cffi_backend/realize_c_type.py
@@ -83,6 +83,7 @@
         self.space = space
         self.all_primitives = [None] * cffi_opcode._NUM_PRIM
         self.file_struct = None
+        self.rec_level = 0
 
     def get_file_struct(self):
         if self.file_struct is None:
@@ -406,12 +407,18 @@
     if from_ffi and ffi.cached_types[index] is not None:
         return ffi.cached_types[index]
 
-    opcodes[index] = rffi.cast(rffi.VOIDP, 255)
+    realize_cache = ffi.space.fromcache(RealizeCache)
+    if realize_cache.rec_level >= 1000:
+        raise oefmt(ffi.space.w_RuntimeError,
+            "type-building recursion too deep or infinite.  "
+            "This is known to occur e.g. in ``struct s { void(*callable)"
+            "(struct s); }''.  Please report if you get this error and "
+            "really need support for your case.")
+    realize_cache.rec_level += 1
     try:
         x = realize_c_type_or_func_now(ffi, op, opcodes, index)
     finally:
-        if opcodes[index] == rffi.cast(rffi.VOIDP, 255):
-            opcodes[index] = op
+        realize_cache.rec_level -= 1
 
     if from_ffi:
         assert ffi.cached_types[index] is None or ffi.cached_types[index] is x
@@ -461,13 +468,6 @@
                                       'c_type_index')
         x = realize_c_type_or_func(ffi, ffi.ctxobj.ctx.c_types, type_index)
 
-    elif case == 255:
-        raise oefmt(ffi.space.w_RuntimeError,
-            "found a situation in which we try to build a type recursively.  "
-            "This is known to occur e.g. in ``struct s { void(*callable)"
-            "(struct s); }''.  Please report if you get this error and "
-            "really need support for your case.")
-
     else:
         raise oefmt(ffi.space.w_NotImplementedError, "op=%d", case)
 
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.13.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.13.1", ("This test_c.py file is for testing a version"
                                  " of cffi that differs from the one that we"
                                  " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
diff --git a/pypy/module/_cffi_backend/test/test_re_python.py b/pypy/module/_cffi_backend/test/test_re_python.py
--- a/pypy/module/_cffi_backend/test/test_re_python.py
+++ b/pypy/module/_cffi_backend/test/test_re_python.py
@@ -74,6 +74,7 @@
         struct foo_s;
         typedef struct bar_s { int x; signed char a[]; } bar_t;
         enum foo_e { AA, BB, CC };
+        typedef struct selfref { struct selfref *next; } *selfref_ptr_t;
         """)
         ffi.set_source('re_python_pysrc', None)
         ffi.emit_python_code(str(tmpdir.join('re_python_pysrc.py')))
@@ -238,3 +239,9 @@
             "foobar", _version=0x2594)
         assert str(e.value).startswith(
             "cffi out-of-line Python module 'foobar' has unknown version")
+
+    def test_selfref(self):
+        # based on cffi issue #429
+        self.fix_path()
+        from re_python_pysrc import ffi
+        ffi.new("selfref_ptr_t")
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -2156,8 +2156,8 @@
             };
         """)
         e = raises(RuntimeError, ffi.new, "struct BinaryTree *")
-        assert str(e.value) == (
-            "found a situation in which we try to build a type recursively.  "
-            "This is known to occur e.g. in ``struct s { void(*callable)"
-            "(struct s); }''.  Please report if you get this error and "
-            "really need support for your case.")
+        # we should check e.value, but untranslated it crashes with a
+        # regular recursion error.  There is a chance it occurs translated
+        # too, but likely the check in the code ">= 1000" usually triggers
+        # before that, and raise a RuntimeError too, but with the more
+        # explicit message.
diff --git a/rpython/translator/platform/darwin.py b/rpython/translator/platform/darwin.py
--- a/rpython/translator/platform/darwin.py
+++ b/rpython/translator/platform/darwin.py
@@ -99,6 +99,7 @@
                      no_precompile_cfiles = [], profopt=False, config=None):
         # ensure frameworks are passed in the Makefile
         fs = self._frameworks(eci.frameworks)
+        extra_libs = self.extra_libs
         if len(fs) > 0:
             # concat (-framework, FrameworkName) pairs
             self.extra_libs += tuple(map(" ".join, zip(fs[::2], fs[1::2])))
@@ -107,6 +108,7 @@
                                 headers_to_precompile=headers_to_precompile,
                                 no_precompile_cfiles = no_precompile_cfiles,
                                 profopt=profopt, config=config)
+        self.extra_libs = extra_libs
         return mk
 
 class Darwin_PowerPC(Darwin):#xxx fixme, mwp

From pypy.commits at gmail.com  Tue Oct 22 03:43:50 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 22 Oct 2019 00:43:50 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: skip flaky test on darwin too
Message-ID: <5daeb336.1c69fb81.13645.389d@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r97831:ce5b78b5c182
Date: 2019-10-22 10:42 +0300
http://bitbucket.org/pypy/pypy/changeset/ce5b78b5c182/

Log:	skip flaky test on darwin too

diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py
--- a/pypy/module/cpyext/test/test_eval.py
+++ b/pypy/module/cpyext/test/test_eval.py
@@ -344,7 +344,7 @@
                     return module.get_flags()""", ns)
         assert ns['nested_flags']() == (0, 0)
 
-    @pytest.mark.xfail("sys.platform == 'win32'", reason='Hangs the process', run=False)
+    @pytest.mark.xfail("'linux' not in sys.platform", reason='Hangs the process', run=False)
     def test_recursive_function(self):
         module = self.import_extension('foo', [
             ("call_recursive", "METH_NOARGS",

From pypy.commits at gmail.com  Tue Oct 22 04:11:25 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 22 Oct 2019 01:11:25 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: py_w_stopcode is only for "ifndef
 PYPY_VERSION"
Message-ID: <5daeb9ad.1c69fb81.cd53b.461f@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r97832:b4b11d42cd34
Date: 2019-10-22 11:10 +0300
http://bitbucket.org/pypy/pypy/changeset/b4b11d42cd34/

Log:	py_w_stopcode is only for "ifndef PYPY_VERSION"

diff --git a/lib_pypy/_testcapimodule.c b/lib_pypy/_testcapimodule.c
--- a/lib_pypy/_testcapimodule.c
+++ b/lib_pypy/_testcapimodule.c
@@ -4544,10 +4544,10 @@
     {"pyobject_fastcalldict", test_pyobject_fastcalldict, METH_VARARGS},
     {"pyobject_fastcallkeywords", test_pyobject_fastcallkeywords, METH_VARARGS},
     {"raise_SIGINT_then_send_None", raise_SIGINT_then_send_None, METH_VARARGS},
-#endif
 #ifdef W_STOPCODE
     {"W_STOPCODE", py_w_stopcode, METH_VARARGS},
 #endif
+#endif
     {NULL, NULL} /* sentinel */
 };
 

From pypy.commits at gmail.com  Tue Oct 22 08:06:56 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 22 Oct 2019 05:06:56 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: fix error msg for py3.6
Message-ID: <5daef0e0.1c69fb81.ea1eb.b0ce@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r97833:73df60cabe78
Date: 2019-10-22 11:28 +0300
http://bitbucket.org/pypy/pypy/changeset/73df60cabe78/

Log:	fix error msg for py3.6

diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -1164,7 +1164,7 @@
         # issue bb-3091 crash in BuiltinCodePassThroughArguments0.funcrun
         import sys
         if '__pypy__' in sys.modules:
-            msg_fmt = "%s instance as first argument (got %s"
+            msg_fmt = "'%s' object expected, got '%s'"
         else:
             msg_fmt = "'%s' object but received a '%s'"
         for obj in (dict, set):

From pypy.commits at gmail.com  Tue Oct 22 08:06:59 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 22 Oct 2019 05:06:59 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: fix failing tests
Message-ID: <5daef0e3.1c69fb81.26c1d.3f63@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r97834:90a079f62b2b
Date: 2019-10-22 12:21 +0300
http://bitbucket.org/pypy/pypy/changeset/90a079f62b2b/

Log:	fix failing tests

diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -644,7 +644,7 @@
         import infinite_reload
 
     def test_reload_module_subclass(self):
-        import types
+        import types, imp
 
         #MyModType = types.ModuleType
         class MyModType(types.ModuleType):
@@ -654,7 +654,7 @@
         with raises(ImportError):
             # Fails because the module is not in sys.modules, but *not* because
             # it's a subtype of ModuleType.
-            reload(m)
+            imp.reload(m)
 
 
     def test_explicitly_missing(self):
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -156,7 +156,14 @@
 
     def test_getfilesystemencoding(self):
         import sys
-        assert sys.getfilesystemencoding() == self.filesystemenc
+        enc = sys.getfilesystemencoding()
+        if self.appdirect:
+            assert enc == self.filesystemenc
+        else:
+            # see comment in 'setup_after_space_initialization'
+            untranslated_enc = {'win32': 'mbcs', 'darwin': 'utf-8'}.get(enc, 'ascii')
+            assert enc == untranslated_enc
+            
 
     def test_float_info(self):
         import sys

From pypy.commits at gmail.com  Tue Oct 22 08:07:01 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 22 Oct 2019 05:07:01 -0700 (PDT)
Subject: [pypy-commit] pypy default: remove the _ssl module;
 we now use the cffi-based one
Message-ID: <5daef0e5.1c69fb81.5b8d8.9cda@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r97835:8d3df17fc5ee
Date: 2019-10-22 15:05 +0300
http://bitbucket.org/pypy/pypy/changeset/8d3df17fc5ee/

Log:	remove the _ssl module; we now use the cffi-based one

diff too long, truncating to 2000 out of 3487 lines

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -37,7 +37,7 @@
     "binascii", "_multiprocessing", '_warnings', "_collections",
     "_multibytecodec", "micronumpy", "_continuation", "_cffi_backend",
     "_csv", "_cppyy", "_pypyjson", "_jitlog",
-    #" _ssl", "_hashlib", "crypt"
+    # "_hashlib", "crypt"
 ])
 
 import rpython.rlib.rvmprof.cintf
diff --git a/pypy/module/_ssl/__init__.py b/pypy/module/_ssl/__init__.py
deleted file mode 100644
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
deleted file mode 100644
--- a/pypy/module/_ssl/interp_ssl.py
+++ /dev/null
@@ -1,1833 +0,0 @@
-from rpython.rlib import rpoll, rsocket, rthread, rweakref, rgc
-from rpython.rlib.rarithmetic import intmask, widen, r_uint
-from rpython.rlib.ropenssl import *
-from pypy.module._socket import interp_socket
-from rpython.rlib._rsocket_rffi import MAX_FD_SIZE
-from rpython.rlib.rposix import get_saved_errno
-from rpython.rlib.rweakref import RWeakValueDictionary
-from rpython.rlib.objectmodel import specialize, compute_unique_id
-from rpython.rtyper.lltypesystem import lltype, rffi
-
-from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.error import OperationError, oefmt, wrap_oserror
-from pypy.interpreter.gateway import interp2app, unwrap_spec
-from pypy.interpreter.typedef import TypeDef, GetSetProperty
-from pypy.module._ssl.ssl_data import (
-    LIBRARY_CODES_TO_NAMES, ERROR_CODES_TO_NAMES)
-
-
-# user defined constants
-X509_NAME_MAXLEN = 256
-# these mirror ssl.h
-PY_SSL_ERROR_NONE, PY_SSL_ERROR_SSL = 0, 1
-PY_SSL_ERROR_WANT_READ, PY_SSL_ERROR_WANT_WRITE = 2, 3
-PY_SSL_ERROR_WANT_X509_LOOKUP = 4
-PY_SSL_ERROR_SYSCALL = 5  # look at error stack/return value/errno
-PY_SSL_ERROR_ZERO_RETURN, PY_SSL_ERROR_WANT_CONNECT = 6, 7
-# start of non ssl.h errorcodes
-PY_SSL_ERROR_EOF = 8  # special case of SSL_ERROR_SYSCALL
-PY_SSL_ERROR_INVALID_ERROR_CODE = 9
-
-PY_SSL_CERT_NONE, PY_SSL_CERT_OPTIONAL, PY_SSL_CERT_REQUIRED = 0, 1, 2
-
-PY_SSL_CLIENT, PY_SSL_SERVER = 0, 1
-
-(PY_SSL_VERSION_SSL2, PY_SSL_VERSION_SSL3,
- PY_SSL_VERSION_TLS, PY_SSL_VERSION_TLS1, PY_SSL_VERSION_TLS1_1,
- PY_SSL_VERSION_TLS1_2) = range(6)
-
-
-SOCKET_IS_NONBLOCKING, SOCKET_IS_BLOCKING = 0, 1
-SOCKET_HAS_TIMED_OUT, SOCKET_HAS_BEEN_CLOSED = 2, 3
-SOCKET_TOO_LARGE_FOR_SELECT, SOCKET_OPERATION_OK = 4, 5
-
-HAVE_RPOLL = 'poll' in dir(rpoll)
-
-constants = {}
-constants["SSL_ERROR_ZERO_RETURN"] = PY_SSL_ERROR_ZERO_RETURN
-constants["SSL_ERROR_WANT_READ"] = PY_SSL_ERROR_WANT_READ
-constants["SSL_ERROR_WANT_WRITE"] = PY_SSL_ERROR_WANT_WRITE
-constants["SSL_ERROR_WANT_X509_LOOKUP"] = PY_SSL_ERROR_WANT_X509_LOOKUP
-constants["SSL_ERROR_SYSCALL"] = PY_SSL_ERROR_SYSCALL
-constants["SSL_ERROR_SSL"] = PY_SSL_ERROR_SSL
-constants["SSL_ERROR_WANT_CONNECT"] = PY_SSL_ERROR_WANT_CONNECT
-constants["SSL_ERROR_EOF"] = PY_SSL_ERROR_EOF
-constants["SSL_ERROR_INVALID_ERROR_CODE"] = PY_SSL_ERROR_INVALID_ERROR_CODE
-
-constants["CERT_NONE"]     = PY_SSL_CERT_NONE
-constants["CERT_OPTIONAL"] = PY_SSL_CERT_OPTIONAL
-constants["CERT_REQUIRED"] = PY_SSL_CERT_REQUIRED
-
-constants["VERIFY_DEFAULT"] = 0
-constants["VERIFY_CRL_CHECK_LEAF"] = X509_V_FLAG_CRL_CHECK
-constants["VERIFY_CRL_CHECK_CHAIN"] = X509_V_FLAG_CRL_CHECK|X509_V_FLAG_CRL_CHECK_ALL
-constants["VERIFY_X509_STRICT"] = X509_V_FLAG_X509_STRICT
-
-constants["HAS_SNI"] = HAS_SNI
-constants["HAS_TLS_UNIQUE"] = HAVE_OPENSSL_FINISHED
-constants["HAS_ECDH"] = not OPENSSL_NO_ECDH
-constants["HAS_NPN"] = HAS_NPN
-constants["HAS_ALPN"] = HAS_ALPN
-
-constants["PROTOCOL_TLS"] = PY_SSL_VERSION_TLS
-constants["PROTOCOL_SSLv23"] = PY_SSL_VERSION_TLS  # Legacy name
-if not OPENSSL_NO_SSL2:
-    constants["PROTOCOL_SSLv2"]  = PY_SSL_VERSION_SSL2
-if not OPENSSL_NO_SSL3:
-    constants["PROTOCOL_SSLv3"]  = PY_SSL_VERSION_SSL3
-constants["PROTOCOL_TLSv1"]  = PY_SSL_VERSION_TLS1
-if HAVE_TLSv1_2:
-    constants["PROTOCOL_TLSv1_1"] = PY_SSL_VERSION_TLS1_1
-    constants["OP_NO_TLSv1_1"] = SSL_OP_NO_TLSv1_1
-    constants["PROTOCOL_TLSv1_2"] = PY_SSL_VERSION_TLS1_2
-    constants["OP_NO_TLSv1_2"] = SSL_OP_NO_TLSv1_2
-
-constants["OP_ALL"] = SSL_OP_ALL &~SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS
-constants["OP_NO_SSLv2"] = SSL_OP_NO_SSLv2
-constants["OP_NO_SSLv3"] = SSL_OP_NO_SSLv3
-constants["OP_NO_TLSv1"] = SSL_OP_NO_TLSv1
-constants["OP_CIPHER_SERVER_PREFERENCE"] = SSL_OP_CIPHER_SERVER_PREFERENCE
-constants["OP_SINGLE_DH_USE"] = SSL_OP_SINGLE_DH_USE
-constants["OP_SINGLE_ECDH_USE"] = SSL_OP_SINGLE_ECDH_USE
-if SSL_OP_NO_COMPRESSION is not None:
-    constants["OP_NO_COMPRESSION"] = SSL_OP_NO_COMPRESSION
-
-constants["OPENSSL_VERSION_NUMBER"] = OPENSSL_VERSION_NUMBER
-ver = OPENSSL_VERSION_NUMBER
-ver, status = divmod(ver, 16)
-ver, patch  = divmod(ver, 256)
-ver, fix    = divmod(ver, 256)
-ver, minor  = divmod(ver, 256)
-ver, major  = divmod(ver, 256)
-version_info = (major, minor, fix, patch, status)
-constants["OPENSSL_VERSION_INFO"] = version_info
-constants["_OPENSSL_API_VERSION"] = version_info
-constants["OPENSSL_VERSION"] = SSLEAY_VERSION
-
-
-def ssl_error(space, msg, errno=0, w_errtype=None, errcode=0):
-    reason_str = None
-    lib_str = None
-    if errcode:
-        err_lib = libssl_ERR_GET_LIB(errcode)
-        err_reason = libssl_ERR_GET_REASON(errcode)
-        reason_str = ERROR_CODES_TO_NAMES.get((err_lib, err_reason), None)
-        lib_str = LIBRARY_CODES_TO_NAMES.get(err_lib, None)
-        raw_msg = libssl_ERR_reason_error_string(errcode)
-        msg = None
-        if raw_msg:
-            msg = rffi.charp2str(raw_msg)
-    if not msg:
-        msg = "unknown error"
-    if reason_str and lib_str:
-        msg = "[%s: %s] %s" % (lib_str, reason_str, msg)
-    elif lib_str:
-        msg = "[%s] %s" % (lib_str, msg)
-
-    w_exception_class = w_errtype or get_exception_class(space, 'w_sslerror')
-    if errno or errcode:
-        w_exception = space.call_function(w_exception_class,
-                                          space.newint(errno), space.newtext(msg))
-    else:
-        w_exception = space.call_function(w_exception_class, space.newtext(msg))
-    space.setattr(w_exception, space.newtext("reason"),
-                  space.newtext_or_none(reason_str))
-    space.setattr(w_exception, space.newtext("library"),
-                  space.newtext_or_none(lib_str))
-    return OperationError(w_exception_class, w_exception)
-
-class SSLNpnProtocols(object):
-
-    def __init__(self, ctx, protos):
-        self.protos = protos
-        self.buf, self.llobj, self.bufflag = rffi.get_nonmovingbuffer_ll(protos)
-        NPN_STORAGE.set(rffi.cast(lltype.Unsigned, self.buf), self)
-
-        # set both server and client callbacks, because the context
-        # can be used to create both types of sockets
-        libssl_SSL_CTX_set_next_protos_advertised_cb(
-            ctx, self.advertiseNPN_cb, self.buf)
-        libssl_SSL_CTX_set_next_proto_select_cb(
-            ctx, self.selectNPN_cb, self.buf)
-
-    def __del__(self):
-        rffi.free_nonmovingbuffer_ll(
-            self.buf, self.llobj, self.bufflag)
-
-    @staticmethod
-    def advertiseNPN_cb(s, data_ptr, len_ptr, args):
-        npn = NPN_STORAGE.get(rffi.cast(lltype.Unsigned, args))
-        if npn and npn.protos:
-            data_ptr[0] = npn.buf
-            len_ptr[0] = rffi.cast(rffi.UINT, len(npn.protos))
-        else:
-            data_ptr[0] = lltype.nullptr(rffi.CCHARP.TO)
-            len_ptr[0] = rffi.cast(rffi.UINT, 0)
-
-        return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_OK)
-
-    @staticmethod
-    def selectNPN_cb(s, out_ptr, outlen_ptr, server, server_len, args):
-        npn = NPN_STORAGE.get(rffi.cast(lltype.Unsigned, args))
-        if npn and npn.protos:
-            client = npn.buf
-            client_len = len(npn.protos)
-        else:
-            client = lltype.nullptr(rffi.CCHARP.TO)
-            client_len = 0
-
-        libssl_SSL_select_next_proto(out_ptr, outlen_ptr,
-                                     server, server_len,
-                                     client, client_len)
-        return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_OK)
-
-
-class SSLAlpnProtocols(object):
-
-    def __init__(self, ctx, protos):
-        self.protos = protos
-        self.buf, self.llobj, self.bufflag = rffi.get_nonmovingbuffer_ll(protos)
-        ALPN_STORAGE.set(rffi.cast(lltype.Unsigned, self.buf), self)
-
-        with rffi.scoped_str2charp(protos) as protos_buf:
-            if libssl_SSL_CTX_set_alpn_protos(
-                    ctx, rffi.cast(rffi.UCHARP, protos_buf), len(protos)):
-                raise MemoryError
-        libssl_SSL_CTX_set_alpn_select_cb(
-            ctx, self.selectALPN_cb, self.buf)
-
-    def __del__(self):
-        rffi.free_nonmovingbuffer_ll(
-            self.buf, self.llobj, self.bufflag)
-
-    @staticmethod
-    def selectALPN_cb(s, out_ptr, outlen_ptr, client, client_len, args):
-        alpn = ALPN_STORAGE.get(rffi.cast(lltype.Unsigned, args))
-        if alpn and alpn.protos:
-            server = alpn.buf
-            server_len = len(alpn.protos)
-        else:
-            server = lltype.nullptr(rffi.CCHARP.TO)
-            server_len = 0
-
-        ret = libssl_SSL_select_next_proto(out_ptr, outlen_ptr,
-                                           server, server_len,
-                                           client, client_len)
-        if ret != OPENSSL_NPN_NEGOTIATED:
-            return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_NOACK)
-        return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_OK)
-
-
-NPN_STORAGE = RWeakValueDictionary(r_uint, SSLNpnProtocols)
-ALPN_STORAGE = RWeakValueDictionary(r_uint, SSLAlpnProtocols)
-
-SOCKET_STORAGE = RWeakValueDictionary(int, W_Root)
-
-
-if HAVE_OPENSSL_RAND:
-    # helper routines for seeding the SSL PRNG
-    @unwrap_spec(string='bytes', entropy=float)
-    def RAND_add(space, string, entropy):
-        """RAND_add(string, entropy)
-
-
-        Mix string into the OpenSSL PRNG state.  entropy (a float) is a lower
-        bound on the entropy contained in string."""
-        with rffi.scoped_nonmovingbuffer(string) as buf:
-            libssl_RAND_add(buf, len(string), entropy)
-
-    def RAND_status(space):
-        """RAND_status() -> 0 or 1
-
-        Returns 1 if the OpenSSL PRNG has been seeded with enough data
-        and 0 if not.  It is necessary to seed the PRNG with RAND_add()
-        on some platforms before using the ssl() function."""
-
-        res = libssl_RAND_status()
-        return space.newint(res)
-
-    if HAVE_OPENSSL_RAND_EGD:
-        @unwrap_spec(path='text')
-        def RAND_egd(space, path):
-            """RAND_egd(path) -> bytes
-
-            Queries the entropy gather daemon (EGD) on socket path.  Returns number
-            of bytes read.  Raises socket.sslerror if connection to EGD fails or
-            if it does provide enough data to seed PRNG."""
-            with rffi.scoped_str2charp(path) as socket_path:
-                bytes = libssl_RAND_egd(socket_path)
-            if bytes == -1:
-                raise ssl_error(space,
-                                "EGD connection failed or EGD did not return "
-                                "enough data to seed the PRNG")
-            return space.newint(bytes)
-    else:
-        # Dummy func for platforms missing RAND_egd(). Most likely LibreSSL.
-        @unwrap_spec(path='text')
-        def RAND_egd(space, path):
-            raise ssl_error(space, "RAND_egd unavailable")
-
-
-class _SSLSocket(W_Root):
-    @staticmethod
-    def descr_new(space, w_ctx, w_sock, socket_type,
-                  w_hostname=None, w_ssl_sock=None):
-        self = _SSLSocket()
-
-        self.space = space
-        self.w_ctx = w_ctx
-        self.peer_cert = lltype.nullptr(X509.TO)
-        self.shutdown_seen_zero = False
-        self.handshake_done = False
-
-        sock_fd = space.int_w(space.call_method(w_sock, "fileno"))
-        self.ssl = libssl_SSL_new(w_ctx.ctx)  # new ssl struct
-
-        rgc.add_memory_pressure(6 * 1024)
-        self.register_finalizer(space)
-
-        index = compute_unique_id(self)
-        libssl_SSL_set_app_data(self.ssl, rffi.cast(rffi.VOIDP, index))
-        SOCKET_STORAGE.set(index, self)
-
-        libssl_SSL_set_fd(self.ssl, sock_fd)  # set the socket for SSL
-        # The ACCEPT_MOVING_WRITE_BUFFER flag is necessary because the address
-        # of a str object may be changed by the garbage collector.
-        libssl_SSL_set_mode(
-            self.ssl, SSL_MODE_AUTO_RETRY | SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER)
-
-        if not space.is_none(w_hostname):
-            if space.isinstance_w(w_hostname, space.w_unicode):
-                w_hostname = space.call_method(w_hostname, "encode",
-                                               space.newtext("idna"))
-            libssl_SSL_set_tlsext_host_name(
-                self.ssl, space.bytes_w(w_hostname))
-
-        # If the socket is in non-blocking mode or timeout mode, set the BIO
-        # to non-blocking mode (blocking is the default)
-        w_timeout = space.call_method(w_sock, "gettimeout")
-        has_timeout = not space.is_none(w_timeout)
-        if has_timeout:
-            # Set both the read and write BIO's to non-blocking mode
-            libssl_BIO_set_nbio(libssl_SSL_get_rbio(self.ssl), 1)
-            libssl_BIO_set_nbio(libssl_SSL_get_wbio(self.ssl), 1)
-
-        if socket_type == PY_SSL_CLIENT:
-            libssl_SSL_set_connect_state(self.ssl)
-        else:
-            libssl_SSL_set_accept_state(self.ssl)
-
-        self.socket_type = socket_type
-        self.w_socket = w_sock
-        if w_ssl_sock:
-            self.ssl_sock_weakref_w = rweakref.ref(w_ssl_sock)
-        else:
-            self.ssl_sock_weakref_w = None
-        return self
-
-    def _finalize_(self):
-        peer_cert = self.peer_cert
-        if peer_cert:
-            self.peer_cert = lltype.nullptr(X509.TO)
-            libssl_X509_free(peer_cert)
-        ssl = self.ssl
-        if ssl:
-            self.ssl = lltype.nullptr(SSL.TO)
-            libssl_SSL_free(ssl)
-
-    @unwrap_spec(data='bufferstr')
-    def write(self, space, data):
-        """write(s) -> len
-
-        Writes the string s into the SSL object.  Returns the number
-        of bytes written."""
-        self._refresh_nonblocking(space)
-
-        sockstate = checkwait(space, self.w_socket, True)
-        if sockstate == SOCKET_HAS_TIMED_OUT:
-            raise ssl_error(space, "The write operation timed out")
-        elif sockstate == SOCKET_HAS_BEEN_CLOSED:
-            raise ssl_error(space, "Underlying socket has been closed.")
-        elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT:
-            raise ssl_error(space, "Underlying socket too large for select().")
-
-        num_bytes = 0
-        while True:
-            err = 0
-
-            num_bytes = libssl_SSL_write(self.ssl, data, len(data))
-            err = libssl_SSL_get_error(self.ssl, num_bytes)
-
-            if err == SSL_ERROR_WANT_READ:
-                sockstate = checkwait(space, self.w_socket, False)
-            elif err == SSL_ERROR_WANT_WRITE:
-                sockstate = checkwait(space, self.w_socket, True)
-            else:
-                sockstate = SOCKET_OPERATION_OK
-
-            if sockstate == SOCKET_HAS_TIMED_OUT:
-                raise ssl_error(space, "The write operation timed out")
-            elif sockstate == SOCKET_HAS_BEEN_CLOSED:
-                raise ssl_error(space, "Underlying socket has been closed.")
-            elif sockstate == SOCKET_IS_NONBLOCKING:
-                break
-
-            if err == SSL_ERROR_WANT_READ or err == SSL_ERROR_WANT_WRITE:
-                continue
-            else:
-                break
-
-        if num_bytes > 0:
-            return space.newint(num_bytes)
-        else:
-            raise _ssl_seterror(space, self, num_bytes)
-
-    def pending(self, space):
-        """pending() -> count
-
-        Returns the number of already decrypted bytes available for read,
-        pending on the connection."""
-        count = libssl_SSL_pending(self.ssl)
-        if count < 0:
-            raise _ssl_seterror(space, self, count)
-        return space.newint(count)
-
-    @unwrap_spec(num_bytes=int)
-    def read(self, space, num_bytes, w_buffer=None):
-        """read([len]) -> string
-
-        Read up to len bytes from the SSL socket."""
-        count = libssl_SSL_pending(self.ssl)
-        if not count:
-            sockstate = checkwait(space, self.w_socket, False)
-            if sockstate == SOCKET_HAS_TIMED_OUT:
-                raise ssl_error(space, "The read operation timed out")
-            elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT:
-                raise ssl_error(space,
-                                "Underlying socket too large for select().")
-            elif sockstate == SOCKET_HAS_BEEN_CLOSED:
-                if libssl_SSL_get_shutdown(self.ssl) == SSL_RECEIVED_SHUTDOWN:
-                    return space.newbytes('')
-                raise ssl_error(space,
-                                "Socket closed without SSL shutdown handshake")
-
-        if w_buffer:
-            rwbuffer = space.getarg_w('w*', w_buffer)
-            buflen = rwbuffer.getlength()
-            if not 0 < num_bytes <= buflen:
-                num_bytes = buflen
-        else:
-            if num_bytes < 0:
-                raise oefmt(space.w_ValueError, "size should not be negative")
-            rwbuffer = None
-
-        if num_bytes <= 0:
-            if rwbuffer:
-                return space.newint(0)
-            else:
-                return space.newbytes("")
-
-        with rffi.scoped_alloc_buffer(num_bytes) as buf:
-            while True:
-                err = 0
-
-                count = libssl_SSL_read(self.ssl, buf.raw, num_bytes)
-                err = libssl_SSL_get_error(self.ssl, count)
-
-                if err == SSL_ERROR_WANT_READ:
-                    sockstate = checkwait(space, self.w_socket, False)
-                elif err == SSL_ERROR_WANT_WRITE:
-                    sockstate = checkwait(space, self.w_socket, True)
-                elif (err == SSL_ERROR_ZERO_RETURN and
-                        libssl_SSL_get_shutdown(self.ssl) == SSL_RECEIVED_SHUTDOWN):
-                    if rwbuffer:
-                        return space.newint(0)
-                    else:
-                        return space.newbytes("")
-                else:
-                    sockstate = SOCKET_OPERATION_OK
-
-                if sockstate == SOCKET_HAS_TIMED_OUT:
-                    raise ssl_error(space, "The read operation timed out")
-                elif sockstate == SOCKET_IS_NONBLOCKING:
-                    break
-
-                if err == SSL_ERROR_WANT_READ or err == SSL_ERROR_WANT_WRITE:
-                    continue
-                else:
-                    break
-
-            if count <= 0:
-                raise _ssl_seterror(space, self, count)
-
-            result = buf.str(count)
-
-        if rwbuffer:
-            rwbuffer.setslice(0, result)
-            return space.newint(count)
-        else:
-            return space.newbytes(result)
-
-    def _refresh_nonblocking(self, space):
-        # just in case the blocking state of the socket has been changed
-        w_timeout = space.call_method(self.w_socket, "gettimeout")
-        nonblocking = not space.is_w(w_timeout, space.w_None)
-        libssl_BIO_set_nbio(libssl_SSL_get_rbio(self.ssl), nonblocking)
-        libssl_BIO_set_nbio(libssl_SSL_get_wbio(self.ssl), nonblocking)
-
-    def do_handshake(self, space):
-        self._refresh_nonblocking(space)
-
-        # Actually negotiate SSL connection
-        # XXX If SSL_do_handshake() returns 0, it's also a failure.
-        while True:
-            ret = libssl_SSL_do_handshake(self.ssl)
-            err = libssl_SSL_get_error(self.ssl, ret)
-            # XXX PyErr_CheckSignals()
-            if err == SSL_ERROR_WANT_READ:
-                sockstate = checkwait(space, self.w_socket, False)
-            elif err == SSL_ERROR_WANT_WRITE:
-                sockstate = checkwait(space, self.w_socket, True)
-            else:
-                sockstate = SOCKET_OPERATION_OK
-            if sockstate == SOCKET_HAS_TIMED_OUT:
-                raise ssl_error(space, "The handshake operation timed out")
-            elif sockstate == SOCKET_HAS_BEEN_CLOSED:
-                raise ssl_error(space, "Underlying socket has been closed.")
-            elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT:
-                raise ssl_error(space,
-                                "Underlying socket too large for select().")
-            elif sockstate == SOCKET_IS_NONBLOCKING:
-                break
-
-            if err == SSL_ERROR_WANT_READ or err == SSL_ERROR_WANT_WRITE:
-                continue
-            else:
-                break
-
-        if ret <= 0:
-            raise _ssl_seterror(space, self, ret)
-
-        if self.peer_cert:
-            libssl_X509_free(self.peer_cert)
-        self.peer_cert = libssl_SSL_get_peer_certificate(self.ssl)
-        self.handshake_done = True
-
-    def shutdown(self, space):
-        # Guard against closed socket
-        w_fileno = space.call_method(self.w_socket, "fileno")
-        if space.int_w(w_fileno) < 0:
-            raise ssl_error(space, "Underlying socket has been closed")
-
-        self._refresh_nonblocking(space)
-
-        zeros = 0
-        while True:
-            # Disable read-ahead so that unwrap can work correctly.
-            # Otherwise OpenSSL might read in too much data,
-            # eating clear text data that happens to be
-            # transmitted after the SSL shutdown.
-            # Should be safe to call repeatedly everytime this
-            # function is used and the shutdown_seen_zero != 0
-            # condition is met.
-            if self.shutdown_seen_zero:
-                libssl_SSL_set_read_ahead(self.ssl, 0)
-            ret = libssl_SSL_shutdown(self.ssl)
-
-            # if err == 1, a secure shutdown with SSL_shutdown() is complete
-            if ret > 0:
-                break
-            if ret == 0:
-                # Don't loop endlessly; instead preserve legacy
-                # behaviour of trying SSL_shutdown() only twice.
-                # This looks necessary for OpenSSL < 0.9.8m
-                zeros += 1
-                if zeros > 1:
-                    break
-                # Shutdown was sent, now try receiving
-                self.shutdown_seen_zero = True
-                continue
-
-            # Possibly retry shutdown until timeout or failure
-            ssl_err = libssl_SSL_get_error(self.ssl, ret)
-            if ssl_err == SSL_ERROR_WANT_READ:
-                sockstate = checkwait(space, self.w_socket, False)
-            elif ssl_err == SSL_ERROR_WANT_WRITE:
-                sockstate = checkwait(space, self.w_socket, True)
-            else:
-                break
-
-            if sockstate == SOCKET_HAS_TIMED_OUT:
-                if ssl_err == SSL_ERROR_WANT_READ:
-                    raise ssl_error(space, "The read operation timed out")
-                else:
-                    raise ssl_error(space, "The write operation timed out")
-            elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT:
-                raise ssl_error(space,
-                                "Underlying socket too large for select().")
-            elif sockstate != SOCKET_OPERATION_OK:
-                # Retain the SSL error code
-                break
-
-        if ret < 0:
-            raise _ssl_seterror(space, self, ret)
-
-        return self.w_socket
-
-    def cipher(self, space):
-        if not self.ssl:
-            return space.w_None
-        current = libssl_SSL_get_current_cipher(self.ssl)
-        if not current:
-            return space.w_None
-
-        name = libssl_SSL_CIPHER_get_name(current)
-        w_name = space.newtext(rffi.charp2str(name)) if name else space.w_None
-
-        proto = libssl_SSL_CIPHER_get_version(current)
-        w_proto = space.newtext(rffi.charp2str(proto)) if proto else space.w_None
-
-        bits = libssl_SSL_CIPHER_get_bits(current,
-                                          lltype.nullptr(rffi.INTP.TO))
-        w_bits = space.newint(bits)
-        return space.newtuple([w_name, w_proto, w_bits])
-
-    @unwrap_spec(der=bool)
-    def peer_certificate(self, space, der=False):
-        """peer_certificate([der=False]) -> certificate
-
-        Returns the certificate for the peer.  If no certificate was
-        provided, returns None.  If a certificate was provided, but not
-        validated, returns an empty dictionary.  Otherwise returns a
-        dict containing information about the peer certificate.
-
-        If the optional argument is True, returns a DER-encoded copy of
-        the peer certificate, or None if no certificate was provided.
-        This will return the certificate even if it wasn't validated.
-        """
-        if not self.handshake_done:
-            raise oefmt(space.w_ValueError, "handshake not done yet")
-        if not self.peer_cert:
-            return space.w_None
-
-        if der:
-            # return cert in DER-encoded format
-            return _certificate_to_der(space, self.peer_cert)
-        else:
-            verification = libssl_SSL_CTX_get_verify_mode(
-                libssl_SSL_get_SSL_CTX(self.ssl))
-            if not verification & SSL_VERIFY_PEER:
-                return space.newdict()
-            else:
-                return _decode_certificate(space, self.peer_cert)
-
-    def selected_npn_protocol(self, space):
-        if not HAS_NPN:
-            raise oefmt(space.w_NotImplementedError,
-                        "The NPN extension requires OpenSSL 1.0.1 or later.")
-        with lltype.scoped_alloc(rffi.CCHARPP.TO, 1) as out_ptr:
-            with lltype.scoped_alloc(rffi.UINTP.TO, 1) as len_ptr:
-                libssl_SSL_get0_next_proto_negotiated(self.ssl,
-                                                      out_ptr, len_ptr)
-                if out_ptr[0]:
-                    return space.newtext(
-                        rffi.charpsize2str(out_ptr[0], intmask(len_ptr[0])))
-
-    def selected_alpn_protocol(self, space):
-        if not HAS_ALPN:
-            raise oefmt(space.w_NotImplementedError,
-                        "The ALPN extension requires OpenSSL 1.0.2 or later.")
-        with lltype.scoped_alloc(rffi.CCHARPP.TO, 1) as out_ptr:
-            with lltype.scoped_alloc(rffi.UINTP.TO, 1) as len_ptr:
-                libssl_SSL_get0_alpn_selected(self.ssl,
-                                              out_ptr, len_ptr)
-                if out_ptr[0]:
-                    return space.newtext(
-                        rffi.charpsize2str(out_ptr[0], intmask(len_ptr[0])))
-
-    def compression_w(self, space):
-        if not self.ssl:
-            return space.w_None
-        comp_method = libssl_SSL_get_current_compression(self.ssl)
-        if not comp_method:
-            return space.w_None
-        method_type = intmask(libssl_COMP_get_type(comp_method))
-        if method_type == NID_undef:
-            return space.w_None
-        short_name = libssl_COMP_get_name(comp_method)
-        if not short_name:
-            return space.w_None
-        return space.newtext(rffi.charp2str(short_name))
-
-    def version_w(self, space):
-        if not self.ssl:
-            return space.w_None
-        version = libssl_SSL_get_version(self.ssl)
-        if not version:
-            return space.w_None
-        return space.newtext(rffi.charp2str(version))
-
-    def tls_unique_cb_w(self, space):
-        """Returns the 'tls-unique' channel binding data, as defined by RFC 5929.
-        If the TLS handshake is not yet complete, None is returned"""
-
-        # In case of 'tls-unique' it will be 12 bytes for TLS, 36
-        # bytes for older SSL, but let's be safe
-        CB_MAXLEN = 128
-
-        with lltype.scoped_alloc(rffi.CCHARP.TO, CB_MAXLEN) as buf:
-            if (libssl_SSL_session_reused(self.ssl) ^
-                (self.socket_type == PY_SSL_CLIENT)):
-                # if session is resumed XOR we are the client
-                length = libssl_SSL_get_finished(self.ssl, buf, CB_MAXLEN)
-            else:
-                # if a new session XOR we are the server
-                length = libssl_SSL_get_peer_finished(self.ssl, buf, CB_MAXLEN)
-
-            if length > 0:
-                return space.newbytes(rffi.charpsize2str(buf, intmask(length)))
-
-    def descr_get_context(self, space):
-        return self.w_ctx
-
-    def descr_set_context(self, space, w_ctx):
-        ctx = space.interp_w(_SSLContext, w_ctx)
-        if not HAS_SNI:
-            raise oefmt(space.w_NotImplementedError,
-                        "setting a socket's context "
-                        "is not supported by your OpenSSL library")
-        self.w_ctx = w_ctx
-        libssl_SSL_set_SSL_CTX(self.ssl, ctx.ctx)
-
-
-_SSLSocket.typedef = TypeDef(
-    "_ssl._SSLSocket",
-
-    do_handshake=interp2app(_SSLSocket.do_handshake),
-    write=interp2app(_SSLSocket.write),
-    read=interp2app(_SSLSocket.read),
-    pending=interp2app(_SSLSocket.pending),
-    peer_certificate=interp2app(_SSLSocket.peer_certificate),
-    cipher=interp2app(_SSLSocket.cipher),
-    shutdown=interp2app(_SSLSocket.shutdown),
-    selected_npn_protocol = interp2app(_SSLSocket.selected_npn_protocol),
-    selected_alpn_protocol = interp2app(_SSLSocket.selected_alpn_protocol),
-    compression = interp2app(_SSLSocket.compression_w),
-    version = interp2app(_SSLSocket.version_w),
-    tls_unique_cb = interp2app(_SSLSocket.tls_unique_cb_w),
-    context=GetSetProperty(_SSLSocket.descr_get_context,
-                           _SSLSocket.descr_set_context),
-)
-
-def _certificate_to_der(space, certificate):
-    with lltype.scoped_alloc(rffi.CCHARPP.TO, 1) as buf_ptr:
-        buf_ptr[0] = lltype.nullptr(rffi.CCHARP.TO)
-        length = libssl_i2d_X509(certificate, buf_ptr)
-        if length < 0:
-            raise _ssl_seterror(space, None, 0)
-        try:
-            return space.newbytes(rffi.charpsize2str(buf_ptr[0], length))
-        finally:
-            libssl_OPENSSL_free(buf_ptr[0])
-
-def _decode_certificate(space, certificate):
-    w_retval = space.newdict()
-
-    w_peer = _create_tuple_for_X509_NAME(
-        space, libssl_X509_get_subject_name(certificate))
-    space.setitem(w_retval, space.newtext("subject"), w_peer)
-
-    w_issuer = _create_tuple_for_X509_NAME(
-        space, libssl_X509_get_issuer_name(certificate))
-    space.setitem(w_retval, space.newtext("issuer"), w_issuer)
-
-    space.setitem(w_retval, space.newtext("version"),
-                  space.newint(libssl_X509_get_version(certificate) + 1))
-
-    biobuf = libssl_BIO_new(libssl_BIO_s_mem())
-    try:
-
-        libssl_BIO_reset(biobuf)
-        serialNumber = libssl_X509_get_serialNumber(certificate)
-        libssl_i2a_ASN1_INTEGER(biobuf, serialNumber)
-        # should not exceed 20 octets, 160 bits, so buf is big enough
-        with lltype.scoped_alloc(rffi.CCHARP.TO, 100) as buf:
-            length = libssl_BIO_gets(biobuf, buf, 99)
-            if length < 0:
-                raise _ssl_seterror(space, None, length)
-
-            w_serial = space.newtext(rffi.charpsize2str(buf, length))
-        space.setitem(w_retval, space.newtext("serialNumber"), w_serial)
-
-        libssl_BIO_reset(biobuf)
-        notBefore = libssl_X509_get_notBefore(certificate)
-        libssl_ASN1_TIME_print(biobuf, notBefore)
-        with lltype.scoped_alloc(rffi.CCHARP.TO, 100) as buf:
-            length = libssl_BIO_gets(biobuf, buf, 99)
-            if length < 0:
-                raise _ssl_seterror(space, None, length)
-            w_date = space.newtext(rffi.charpsize2str(buf, length))
-        space.setitem(w_retval, space.newtext("notBefore"), w_date)
-
-        libssl_BIO_reset(biobuf)
-        notAfter = libssl_X509_get_notAfter(certificate)
-        libssl_ASN1_TIME_print(biobuf, notAfter)
-        with lltype.scoped_alloc(rffi.CCHARP.TO, 100) as buf:
-            length = libssl_BIO_gets(biobuf, buf, 99)
-            if length < 0:
-                raise _ssl_seterror(space, None, length)
-            w_date = space.newtext(rffi.charpsize2str(buf, length))
-        space.setitem(w_retval, space.newtext("notAfter"), w_date)
-    finally:
-        libssl_BIO_free(biobuf)
-
-    # Now look for subjectAltName
-    w_alt_names = _get_peer_alt_names(space, certificate)
-    if w_alt_names is not space.w_None:
-        space.setitem(w_retval, space.newtext("subjectAltName"), w_alt_names)
-
-    # Authority Information Access: OCSP URIs
-    w_ocsp = _get_aia_uri(space, certificate, NID_ad_OCSP)
-    if not space.is_none(w_ocsp):
-        space.setitem(w_retval, space.newtext("OCSP"), w_ocsp)
-    w_issuers = _get_aia_uri(space, certificate, NID_ad_ca_issuers)
-    if not space.is_none(w_issuers):
-        space.setitem(w_retval, space.newtext("caIssuers"), w_issuers)
-
-    # CDP (CRL distribution points)
-    w_cdp = _get_crl_dp(space, certificate)
-    if not space.is_none(w_cdp):
-        space.setitem(w_retval, space.newtext("crlDistributionPoints"), w_cdp)
-
-    return w_retval
-
-
-def _create_tuple_for_X509_NAME(space, xname):
-    entry_count = libssl_X509_NAME_entry_count(xname)
-    dn_w = []
-    rdn_w = []
-    rdn_level = -1
-    for index in range(entry_count):
-        entry = libssl_X509_NAME_get_entry(xname, index)
-        # check to see if we've gotten to a new RDN
-        entry_level = intmask(libssl_X509_NAME_ENTRY_set(entry))
-        if rdn_level >= 0:
-            if rdn_level != entry_level:
-                # yes, new RDN
-                # add old RDN to DN
-                dn_w.append(space.newtuple(list(rdn_w)))
-                rdn_w = []
-        rdn_level = entry_level
-
-        # Now add this attribute to the current RDN
-        name = libssl_X509_NAME_ENTRY_get_object(entry)
-        value = libssl_X509_NAME_ENTRY_get_data(entry)
-        attr = _create_tuple_for_attribute(space, name, value)
-        rdn_w.append(attr)
-
-    # Now, there is typically a dangling RDN
-    if rdn_w:
-        dn_w.append(space.newtuple(list(rdn_w)))
-    return space.newtuple(list(dn_w))
-
-
-def _get_peer_alt_names(space, certificate):
-    # this code follows the procedure outlined in
-    # OpenSSL's crypto/x509v3/v3_prn.c:X509v3_EXT_print()
-    # function to extract the STACK_OF(GENERAL_NAME),
-    # then iterates through the stack to add the
-    # names.
-
-    if not certificate:
-        return space.w_None
-
-    # get a memory buffer
-    biobuf = libssl_BIO_new(libssl_BIO_s_mem())
-
-    try:
-        alt_names_w = []
-        i = -1
-        while True:
-            i = libssl_X509_get_ext_by_NID(
-                certificate, NID_subject_alt_name, i)
-            if i < 0:
-                break
-
-            # now decode the altName
-            ext = libssl_X509_get_ext(certificate, i)
-            method = libssl_X509V3_EXT_get(ext)
-            if not method:
-                raise ssl_error(space,
-                                "No method for internalizing subjectAltName!'")
-
-            with lltype.scoped_alloc(rffi.CCHARPP.TO, 1) as p_ptr:
-                ext_value = libssl_X509_EXTENSION_get_data(ext)
-                p_ptr[0] = ext_value.c_data
-                length = intmask(ext_value.c_length)
-                null = lltype.nullptr(rffi.VOIDP.TO)
-                if method[0].c_it:
-                    names = rffi.cast(GENERAL_NAMES, libssl_ASN1_item_d2i(
-                        null, p_ptr, length,
-                        libssl_ASN1_ITEM_ptr(method[0].c_it)))
-                else:
-                    names = rffi.cast(GENERAL_NAMES, method[0].c_d2i(
-                        null, p_ptr, length))
-
-            try:
-                for j in range(libssl_sk_GENERAL_NAME_num(names)):
-                    # Get a rendering of each name in the set of names
-
-                    name = libssl_sk_GENERAL_NAME_value(names, j)
-                    gntype = intmask(name.c_type)
-                    if gntype == GEN_DIRNAME:
-                        # we special-case DirName as a tuple of tuples of
-                        # attributes
-                        dirname = libssl_pypy_GENERAL_NAME_dirn(name)
-                        w_t = space.newtuple([
-                            space.newtext("DirName"),
-                            _create_tuple_for_X509_NAME(space, dirname)
-                            ])
-                    elif gntype in (GEN_EMAIL, GEN_DNS, GEN_URI):
-                        # GENERAL_NAME_print() doesn't handle NULL bytes in
-                        # ASN1_string correctly, CVE-2013-4238
-                        if gntype == GEN_EMAIL:
-                            v = space.newtext("email")
-                        elif gntype == GEN_DNS:
-                            v = space.newtext("DNS")
-                        elif gntype == GEN_URI:
-                            v = space.newtext("URI")
-                        else:
-                            assert False
-                        as_ = libssl_pypy_GENERAL_NAME_dirn(name)
-                        as_ = rffi.cast(ASN1_STRING, as_)
-                        buf = libssl_ASN1_STRING_data(as_)
-                        length = libssl_ASN1_STRING_length(as_)
-                        w_t = space.newtuple([
-                            v, space.newtext(rffi.charpsize2str(buf, length))])
-                    elif gntype == GEN_RID:
-                        with lltype.scoped_alloc(rffi.CCHARP.TO, 2048) as buf:
-                            d_rid = libssl_pypy_GENERAL_NAME_rid(name)
-                            length = libssl_i2t_ASN1_OBJECT(buf, 2047, d_rid)
-                            if length < 0:
-                                raise _ssl_seterror(space, None, 0)
-                            elif length >= 2048:
-                                v = "<INVALID>"
-                            else:
-                                v = rffi.charpsize2str(buf, length)
-                        w_t = space.newtuple([
-                            space.newtext("Registered ID"),
-                            space.call_function(space.w_unicode,
-                                                space.newtext(v))])
-                    else:
-                        # for everything else, we use the OpenSSL print form
-                        if gntype not in (GEN_OTHERNAME, GEN_X400, GEN_EDIPARTY,
-                                          GEN_IPADD):
-                            space.warn(space.newtext("Unknown general name type"),
-                                       space.w_RuntimeWarning)
-                        libssl_BIO_reset(biobuf)
-                        libssl_GENERAL_NAME_print(biobuf, name)
-                        with lltype.scoped_alloc(rffi.CCHARP.TO, 2048) as buf:
-                            length = libssl_BIO_gets(biobuf, buf, 2047)
-                            if length < 0:
-                                raise _ssl_seterror(space, None, 0)
-
-                            v = rffi.charpsize2str(buf, length)
-                        if ':' not in v:
-                            raise oefmt(space.w_ValueError,
-                                        "Invalid value %s", v)
-                        v1, v2 = v.split(':', 1)
-                        w_t = space.newtuple([space.newtext(v1),
-                                              space.newtext(v2)])
-
-                    alt_names_w.append(w_t)
-            finally:
-                libssl_pypy_GENERAL_NAME_pop_free(names)
-    finally:
-        libssl_BIO_free(biobuf)
-
-    if alt_names_w:
-        return space.newtuple(list(alt_names_w))
-    else:
-        return space.w_None
-
-
-def _create_tuple_for_attribute(space, name, value):
-    with lltype.scoped_alloc(rffi.CCHARP.TO, X509_NAME_MAXLEN) as buf:
-        length = libssl_OBJ_obj2txt(buf, X509_NAME_MAXLEN, name, 0)
-        if length < 0:
-            raise _ssl_seterror(space, None, 0)
-        w_name = space.newtext(rffi.charpsize2str(buf, length))
-
-    with lltype.scoped_alloc(rffi.CCHARPP.TO, 1) as buf_ptr:
-        length = libssl_ASN1_STRING_to_UTF8(buf_ptr, value)
-        if length < 0:
-            raise _ssl_seterror(space, None, 0)
-        try:
-            w_value = space.newbytes(rffi.charpsize2str(buf_ptr[0], length))
-            w_value = space.call_method(w_value, "decode", space.newtext("utf-8"))
-        finally:
-            libssl_OPENSSL_free(buf_ptr[0])
-
-    return space.newtuple([w_name, w_value])
-
-
-def _get_aia_uri(space, certificate, nid):
-    info = rffi.cast(AUTHORITY_INFO_ACCESS, libssl_X509_get_ext_d2i(
-        certificate, NID_info_access, None, None))
-    try:
-        if not info or libssl_sk_ACCESS_DESCRIPTION_num(info) == 0:
-            return
-
-        result_w = []
-        for i in range(libssl_sk_ACCESS_DESCRIPTION_num(info)):
-            ad = libssl_sk_ACCESS_DESCRIPTION_value(info, i)
-            if libssl_OBJ_obj2nid(ad[0].c_method) != nid:
-                continue
-
-            name = ad[0].c_location
-            gntype = intmask(name.c_type)
-            if gntype != GEN_URI:
-                continue
-            uri = libssl_pypy_GENERAL_NAME_uri(name)
-            length = intmask(uri.c_length)
-            s_uri = rffi.charpsize2str(uri.c_data, length)
-            result_w.append(space.newtext(s_uri))
-        return space.newtuple(result_w[:])
-    finally:
-        libssl_AUTHORITY_INFO_ACCESS_free(info)
-
-def _get_crl_dp(space, certificate):
-    dps = rffi.cast(stack_st_DIST_POINT, libssl_X509_get_ext_d2i(
-        certificate, NID_crl_distribution_points, None, None))
-    if not dps:
-        return None
-
-    try:
-        cdp_w = []
-        for i in range(libssl_sk_DIST_POINT_num(dps)):
-            dp = libssl_sk_DIST_POINT_value(dps, i)
-            gns = libssl_pypy_DIST_POINT_fullname(dp)
-
-            for j in range(libssl_sk_GENERAL_NAME_num(gns)):
-                name = libssl_sk_GENERAL_NAME_value(gns, j)
-                gntype = intmask(name.c_type)
-                if gntype != GEN_URI:
-                    continue
-                uri = libssl_pypy_GENERAL_NAME_uri(name)
-                length = intmask(uri.c_length)
-                s_uri = rffi.charpsize2str(uri.c_data, length)
-                cdp_w.append(space.newtext(s_uri))
-    finally:
-        libssl_CRL_DIST_POINTS_free(dps)
-    return space.newtuple(cdp_w[:])
-
-def checkwait(space, w_sock, writing):
-    """If the socket has a timeout, do a select()/poll() on the socket.
-    The argument writing indicates the direction.
-    Returns one of the possibilities in the timeout_state enum (above)."""
-
-    w_timeout = space.call_method(w_sock, "gettimeout")
-    if space.is_w(w_timeout, space.w_None):
-        return SOCKET_IS_BLOCKING
-    elif space.float_w(w_timeout) == 0.0:
-        return SOCKET_IS_NONBLOCKING
-    sock_timeout = space.float_w(w_timeout)
-
-    sock_fd = space.int_w(space.call_method(w_sock, "fileno"))
-
-    # guard against closed socket
-    if sock_fd < 0:
-        return SOCKET_HAS_BEEN_CLOSED
-
-    # see if the socket is ready
-
-    # Prefer poll, if available, since you can poll() any fd
-    # which can't be done with select().
-    if HAVE_RPOLL:
-        if writing:
-            fddict = {sock_fd: rpoll.POLLOUT}
-        else:
-            fddict = {sock_fd: rpoll.POLLIN}
-
-        # socket's timeout is in seconds, poll's timeout in ms
-        timeout = int(sock_timeout * 1000 + 0.5)
-        try:
-            ready = rpoll.poll(fddict, timeout)
-        except rpoll.PollError as e:
-            message = e.get_msg()
-            raise ssl_error(space, message, e.errno)
-    else:
-        if MAX_FD_SIZE is not None and sock_fd >= MAX_FD_SIZE:
-            return SOCKET_TOO_LARGE_FOR_SELECT
-
-        try:
-            if writing:
-                r, w, e = rpoll.select([], [sock_fd], [], sock_timeout)
-                ready = w
-            else:
-                r, w, e = rpoll.select([sock_fd], [], [], sock_timeout)
-                ready = r
-        except rpoll.SelectError as e:
-            message = e.get_msg()
-            raise ssl_error(space, message, e.errno)
-    if ready:
-        return SOCKET_OPERATION_OK
-    else:
-        return SOCKET_HAS_TIMED_OUT
-
-
-def _ssl_seterror(space, ss, ret):
-    assert ret <= 0
-
-    errcode = libssl_ERR_peek_last_error()
-
-    if ss is None:
-        return ssl_error(space, None, errcode=errcode)
-    elif ss.ssl:
-        err = libssl_SSL_get_error(ss.ssl, ret)
-    else:
-        err = SSL_ERROR_SSL
-    w_errtype = None
-    errstr = ""
-    errval = 0
-
-    if err == SSL_ERROR_ZERO_RETURN:
-        w_errtype = get_exception_class(space, 'w_sslzeroreturnerror')
-        errstr = "TLS/SSL connection has been closed"
-        errval = PY_SSL_ERROR_ZERO_RETURN
-    elif err == SSL_ERROR_WANT_READ:
-        w_errtype = get_exception_class(space, 'w_sslwantreaderror')
-        errstr = "The operation did not complete (read)"
-        errval = PY_SSL_ERROR_WANT_READ
-    elif err == SSL_ERROR_WANT_WRITE:
-        w_errtype = get_exception_class(space, 'w_sslwantwriteerror')
-        errstr = "The operation did not complete (write)"
-        errval = PY_SSL_ERROR_WANT_WRITE
-    elif err == SSL_ERROR_WANT_X509_LOOKUP:
-        errstr = "The operation did not complete (X509 lookup)"
-        errval = PY_SSL_ERROR_WANT_X509_LOOKUP
-    elif err == SSL_ERROR_WANT_CONNECT:
-        errstr = "The operation did not complete (connect)"
-        errval = PY_SSL_ERROR_WANT_CONNECT
-    elif err == SSL_ERROR_SYSCALL:
-        e = libssl_ERR_get_error()
-        if e == 0:
-            if ret == 0 or space.is_w(ss.w_socket, space.w_None):
-                errstr = "EOF occurred in violation of protocol"
-                errval = PY_SSL_ERROR_EOF
-            elif ret == -1:
-                # the underlying BIO reported an I/0 error
-                error = rsocket.last_error()
-                return interp_socket.converted_error(space, error)
-            else:
-                w_errtype = get_exception_class(space, 'w_sslsyscallerror')
-                errstr = "Some I/O error occurred"
-                errval = PY_SSL_ERROR_SYSCALL
-        else:
-            errstr = rffi.charp2str(libssl_ERR_error_string(e, None))
-            errval = PY_SSL_ERROR_SYSCALL
-    elif err == SSL_ERROR_SSL:
-        errval = PY_SSL_ERROR_SSL
-        if errcode != 0:
-            errstr = rffi.charp2str(libssl_ERR_error_string(errcode, None))
-        else:
-            errstr = "A failure in the SSL library occurred"
-    else:
-        errstr = "Invalid error code"
-        errval = PY_SSL_ERROR_INVALID_ERROR_CODE
-
-    return ssl_error(space, errstr, errval, w_errtype=w_errtype,
-                     errcode=errcode)
-
-def SSLError_descr_str(space, w_exc):
-    w_strerror = space.getattr(w_exc, space.newtext("strerror"))
-    if not space.is_none(w_strerror):
-        return w_strerror
-    return space.str(space.getattr(w_exc, space.newtext("args")))
-
-
-class Cache:
-    def __init__(self, space):
-        w_socketerror = interp_socket.get_error(space, "error")
-        self.w_sslerror = space.new_exception_class(
-            "_ssl.SSLError", w_socketerror)
-        space.setattr(self.w_sslerror, space.newtext('__str__'),
-                      interp2app(SSLError_descr_str).spacebind(space))
-        self.w_sslzeroreturnerror = space.new_exception_class(
-            "_ssl.SSLZeroReturnError", self.w_sslerror)
-        self.w_sslwantreaderror = space.new_exception_class(
-            "_ssl.SSLWantReadError", self.w_sslerror)
-        self.w_sslwantwriteerror = space.new_exception_class(
-            "_ssl.SSLWantWriteError", self.w_sslerror)
-        self.w_sslsyscallerror = space.new_exception_class(
-            "_ssl.SSLSyscallError", self.w_sslerror)
-        self.w_ssleoferror = space.new_exception_class(
-            "_ssl.SSLEOFError", self.w_sslerror)
-
-
- at specialize.memo()
-def get_exception_class(space, name):
-    return getattr(space.fromcache(Cache), name)
-
-
- at unwrap_spec(filename='text')
-def _test_decode_cert(space, filename):
-    cert = libssl_BIO_new(libssl_BIO_s_file())
-    if not cert:
-        raise ssl_error(space, "Can't malloc memory to read file")
-
-    try:
-        if libssl_BIO_read_filename(cert, filename) <= 0:
-            raise ssl_error(space, "Can't open file")
-
-        x = libssl_PEM_read_bio_X509_AUX(cert, None, None, None)
-        if not x:
-            raise ssl_error(space, "Error decoding PEM-encoded file")
-
-        try:
-            return _decode_certificate(space, x)
-        finally:
-            libssl_X509_free(x)
-    finally:
-        libssl_BIO_free(cert)
-
-
-# Data structure for the password callbacks
-class PasswordInfo(object):
-    w_callable = None
-    password = None
-    operationerror = None
-PWINFO_STORAGE = {}
-
-def _password_callback(buf, size, rwflag, userdata):
-    index = rffi.cast(lltype.Signed, userdata)
-    pw_info = PWINFO_STORAGE.get(index, None)
-    if not pw_info:
-        return rffi.cast(rffi.INT, -1)
-    space = pw_info.space
-    password = ""
-    if pw_info.w_callable:
-        try:
-            w_result = space.call_function(pw_info.w_callable)
-            try:
-                password = pw_info.space.bufferstr_w(w_result)
-            except OperationError as e:
-                if not e.match(space, space.w_TypeError):
-                    raise
-                raise oefmt(space.w_TypeError,
-                            "password callback must return a string")
-        except OperationError as e:
-            pw_info.operationerror = e
-            return rffi.cast(rffi.INT, -1)
-    else:
-        password = pw_info.password
-    size = widen(size)
-    if len(password) > size:
-        pw_info.operationerror = oefmt(
-            space.w_ValueError,
-            "password cannot be longer than %d bytes", size)
-        return rffi.cast(rffi.INT, -1)
-    for i, c in enumerate(password):
-        buf[i] = c
-    return rffi.cast(rffi.INT, len(password))
-
-class ServernameCallback(object):
-    w_ctx = None
-    space = None
-SERVERNAME_CALLBACKS = RWeakValueDictionary(int, ServernameCallback)
-
-def _servername_callback(ssl, ad, arg):
-    struct = SERVERNAME_CALLBACKS.get(rffi.cast(lltype.Signed, arg))
-    w_ctx = struct.w_ctx
-    space = struct.space
-    # annotation: ensures that we ignore the case 'space is None'
-    # (it would propagate to a few methods like errorstr())
-    assert w_ctx is not None and space is not None
-    w_callback = struct.w_set_hostname
-    if not w_ctx.servername_callback:
-        # Possible race condition.
-        return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_OK)
-    # The high-level ssl.SSLSocket object
-    index = rffi.cast(lltype.Signed, libssl_SSL_get_app_data(ssl))
-    w_ssl = SOCKET_STORAGE.get(index)
-    assert isinstance(w_ssl, _SSLSocket)
-    if w_ssl.ssl_sock_weakref_w is not None:
-        w_ssl_socket = w_ssl.ssl_sock_weakref_w()
-    else:
-        w_ssl_socket = space.w_None
-    if space.is_none(w_ssl_socket):
-        ad[0] = rffi.cast(rffi.INT, SSL_AD_INTERNAL_ERROR)
-        return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_ALERT_FATAL)
-
-    servername = libssl_SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name)
-    try:
-        if not servername:
-            w_result = space.call_function(w_callback,
-                                           w_ssl_socket, space.w_None, w_ctx)
-
-        else:
-            w_servername = space.newbytes(rffi.charp2str(servername))
-            try:
-                w_servername_idna = space.call_method(
-                    w_servername, 'decode', space.newtext('idna'))
-            except OperationError as e:
-                e.write_unraisable(space, "undecodable server name")
-                ad[0] = rffi.cast(rffi.INT, SSL_AD_INTERNAL_ERROR)
-                return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_ALERT_FATAL)
-
-            w_result = space.call_function(w_callback,
-                                           w_ssl_socket,
-                                           w_servername_idna, w_ctx)
-    except OperationError as e:
-        e.write_unraisable(space, "in servername callback")
-        ad[0] = rffi.cast(rffi.INT, SSL_AD_HANDSHAKE_FAILURE)
-        return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_ALERT_FATAL)
-
-    if space.is_none(w_result):
-        return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_OK)
-    else:
-        try:
-            ad[0] = rffi.cast(rffi.INT, space.int_w(w_result))
-        except OperationError as e:
-            e.write_unraisable(space, "servername callback result")
-            ad[0] = rffi.cast(rffi.INT, SSL_AD_INTERNAL_ERROR)
-        return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_ALERT_FATAL)
-
-
-class _SSLContext(W_Root):
-    @staticmethod
-    @unwrap_spec(protocol=int)
-    def descr_new(space, w_subtype, protocol):
-        if protocol == PY_SSL_VERSION_TLS:
-            method = libssl_TLS_method()
-        elif protocol == PY_SSL_VERSION_TLS1:
-            method = libssl_TLSv1_method()
-        elif protocol == PY_SSL_VERSION_SSL3 and not OPENSSL_NO_SSL3:
-            method = libssl_SSLv3_method()
-        elif protocol == PY_SSL_VERSION_SSL2 and not OPENSSL_NO_SSL2:
-            method = libssl_SSLv2_method()
-        elif protocol == PY_SSL_VERSION_TLS1_1 and HAVE_TLSv1_2:
-            method = libssl_TLSv1_1_method()
-        elif protocol == PY_SSL_VERSION_TLS1_2 and HAVE_TLSv1_2:
-            method = libssl_TLSv1_2_method()
-        else:
-            raise oefmt(space.w_ValueError, "invalid protocol version")
-        ctx = libssl_SSL_CTX_new(method)
-        if not ctx:
-            raise ssl_error(space, "failed to allocate SSL context")
-
-        self = space.allocate_instance(_SSLContext, w_subtype)
-        assert isinstance(self, _SSLContext)
-        rgc.add_memory_pressure(10 * 1024 * 1024, self)
-        self.ctx = ctx
-        self.check_hostname = False
-        self.register_finalizer(space)
-        options = SSL_OP_ALL & ~SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS
-        if protocol != PY_SSL_VERSION_SSL2:
-            options |= SSL_OP_NO_SSLv2
-        if protocol != PY_SSL_VERSION_SSL3:
-            options |= SSL_OP_NO_SSLv3
-        libssl_SSL_CTX_set_options(ctx, options)
-
-        if not OPENSSL_NO_ECDH:
-            # Allow automatic ECDH curve selection (on
-            # OpenSSL 1.0.2+), or use prime256v1 by default.
-            # This is Apache mod_ssl's initialization
-            # policy, so we should be safe.
-            # OpenSSL 1.1 has it enabled by default.
-            if libssl_SSL_CTX_set_ecdh_auto:
-                libssl_SSL_CTX_set_ecdh_auto(self.ctx, 1)
-            else:
-                key = libssl_EC_KEY_new_by_curve_name(NID_X9_62_prime256v1)
-                if not key:
-                    raise _ssl_seterror(space, None, 0)
-                try:
-                    libssl_SSL_CTX_set_tmp_ecdh(self.ctx, key)
-                finally:
-                    libssl_EC_KEY_free(key)
-
-        return self
-
-    def _finalize_(self):
-        ctx = self.ctx
-        if ctx:
-            self.ctx = lltype.nullptr(SSL_CTX.TO)
-            libssl_SSL_CTX_free(ctx)
-
-    @unwrap_spec(server_side=int)
-    def descr_wrap_socket(self, space, w_sock, server_side, w_server_hostname=None, w_ssl_sock=None):
-        return _SSLSocket.descr_new(space, self, w_sock, server_side, w_server_hostname, w_ssl_sock)
-
-    @unwrap_spec(cipherlist='text')
-    def descr_set_ciphers(self, space, cipherlist):
-        ret = libssl_SSL_CTX_set_cipher_list(self.ctx, cipherlist)
-        if ret == 0:
-            libssl_ERR_clear_error()
-            raise ssl_error(space, "No cipher can be selected.")
-
-    def session_stats_w(self, space):
-        w_stats = space.newdict()
-        for name, ssl_func in SSL_CTX_STATS:
-            w_value = space.newint(ssl_func(self.ctx))
-            space.setitem_str(w_stats, name, w_value)
-        return w_stats
-
-    def descr_set_default_verify_paths(self, space):
-        if not libssl_SSL_CTX_set_default_verify_paths(self.ctx):
-            raise ssl_error(space, "")
-
-    def descr_get_options(self, space):
-        return space.newlong(libssl_SSL_CTX_get_options(self.ctx))
-
-    def descr_set_options(self, space, w_new_opts):
-        new_opts = space.int_w(w_new_opts)
-        opts = libssl_SSL_CTX_get_options(self.ctx)
-        clear = opts & ~new_opts
-        set = ~opts & new_opts
-        if clear:
-            if HAVE_SSL_CTX_CLEAR_OPTIONS:
-                libssl_SSL_CTX_clear_options(self.ctx, clear)
-            else:
-                raise oefmt(space.w_ValueError,
-                            "can't clear options before OpenSSL 0.9.8m")
-        if set:
-            libssl_SSL_CTX_set_options(self.ctx, set)
-
-    def descr_get_verify_mode(self, space):
-        mode = libssl_SSL_CTX_get_verify_mode(self.ctx)
-        if mode == SSL_VERIFY_NONE:
-            return space.newlong(PY_SSL_CERT_NONE)
-        elif mode == SSL_VERIFY_PEER:
-            return space.newlong(PY_SSL_CERT_OPTIONAL)
-        elif mode == SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT:
-            return space.newlong(PY_SSL_CERT_REQUIRED)
-        raise ssl_error(space, "invalid return value from SSL_CTX_get_verify_mode")
-
-    def descr_set_verify_mode(self, space, w_mode):
-        n = space.int_w(w_mode)
-        if n == PY_SSL_CERT_NONE:
-            mode = SSL_VERIFY_NONE
-        elif n == PY_SSL_CERT_OPTIONAL:
-            mode = SSL_VERIFY_PEER
-        elif n == PY_SSL_CERT_REQUIRED:
-            mode = SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT
-        else:
-            raise oefmt(space.w_ValueError,
-                        "invalid value for verify_mode")
-        if mode == SSL_VERIFY_NONE and self.check_hostname:
-            raise oefmt(space.w_ValueError,
-                        "Cannot set verify_mode to CERT_NONE when "
-                        "check_hostname is enabled.")
-        libssl_SSL_CTX_set_verify(self.ctx, mode, None)
-
-    def descr_get_verify_flags(self, space):
-        store = libssl_SSL_CTX_get_cert_store(self.ctx)
-        param = libssl_X509_STORE_get0_param(store)
-        flags = libssl_X509_VERIFY_PARAM_get_flags(param)
-        return space.newint(flags)
-
-    def descr_set_verify_flags(self, space, w_obj):
-        new_flags = space.int_w(w_obj)
-        store = libssl_SSL_CTX_get_cert_store(self.ctx)
-        param = libssl_X509_STORE_get0_param(store)
-        flags = libssl_X509_VERIFY_PARAM_get_flags(param)
-        flags_clear = flags & ~new_flags
-        flags_set = ~flags & new_flags
-        if flags_clear and not libssl_X509_VERIFY_PARAM_clear_flags(
-                param, flags_clear):
-            raise _ssl_seterror(space, None, 0)
-        if flags_set and not libssl_X509_VERIFY_PARAM_set_flags(
-                param, flags_set):
-            raise _ssl_seterror(space, None, 0)
-
-    def descr_get_check_hostname(self, space):
-        return space.newbool(self.check_hostname)
-
-    def descr_set_check_hostname(self, space, w_obj):
-        check_hostname = space.is_true(w_obj)
-        if check_hostname and libssl_SSL_CTX_get_verify_mode(self.ctx) == SSL_VERIFY_NONE:
-            raise oefmt(space.w_ValueError,
-                        "check_hostname needs a SSL context with either "
-                        "CERT_OPTIONAL or CERT_REQUIRED")
-        self.check_hostname = check_hostname
-
-    def load_cert_chain_w(self, space, w_certfile, w_keyfile=None,
-                          w_password=None):
-        if space.is_none(w_certfile):
-            certfile = None
-        else:
-            certfile = space.text_w(w_certfile)
-        if space.is_none(w_keyfile):
-            keyfile = certfile
-        else:
-            keyfile = space.text_w(w_keyfile)
-        pw_info = PasswordInfo()
-        pw_info.space = space
-        index = -1
-        if not space.is_none(w_password):
-            index = rthread.get_ident()
-            PWINFO_STORAGE[index] = pw_info
-
-            if space.is_true(space.callable(w_password)):
-                pw_info.w_callable = w_password
-            else:
-                try:
-                    pw_info.password = space.bufferstr_w(w_password)
-                except OperationError as e:
-                    if not e.match(space, space.w_TypeError):
-                        raise
-                    raise oefmt(space.w_TypeError,
-                                "password should be a string or callable")
-
-            libssl_SSL_CTX_set_default_passwd_cb(
-                self.ctx, _password_callback)
-            libssl_SSL_CTX_set_default_passwd_cb_userdata(
-                self.ctx, rffi.cast(rffi.VOIDP, index))
-
-        try:
-            ret = libssl_SSL_CTX_use_certificate_chain_file(self.ctx, certfile)
-            if ret != 1:
-                if pw_info.operationerror:
-                    libssl_ERR_clear_error()
-                    raise pw_info.operationerror
-                errno = get_saved_errno()
-                if errno:
-                    libssl_ERR_clear_error()
-                    raise wrap_oserror(space, OSError(errno, ''),
-                                       exception_name = 'w_IOError')
-                else:
-                    raise _ssl_seterror(space, None, -1)
-
-            ret = libssl_SSL_CTX_use_PrivateKey_file(self.ctx, keyfile,
-                                                     SSL_FILETYPE_PEM)
-            if ret != 1:
-                if pw_info.operationerror:
-                    libssl_ERR_clear_error()
-                    raise pw_info.operationerror
-                errno = get_saved_errno()
-                if errno:
-                    libssl_ERR_clear_error()
-                    raise wrap_oserror(space, OSError(errno, ''),
-                                       exception_name = 'w_IOError')
-                else:
-                    raise _ssl_seterror(space, None, -1)
-
-            ret = libssl_SSL_CTX_check_private_key(self.ctx)
-            if ret != 1:
-                raise _ssl_seterror(space, None, -1)
-        finally:
-            if index >= 0:
-                del PWINFO_STORAGE[index]
-            libssl_SSL_CTX_set_default_passwd_cb(
-                self.ctx, lltype.nullptr(pem_password_cb.TO))
-            libssl_SSL_CTX_set_default_passwd_cb_userdata(
-                self.ctx, None)
-
-    @unwrap_spec(filepath='text')
-    def load_dh_params_w(self, space, filepath):
-        bio = libssl_BIO_new_file(filepath, "r")
-        if not bio:
-            errno = get_saved_errno()
-            libssl_ERR_clear_error()
-            raise wrap_oserror(space, OSError(errno, ''),
-                               exception_name = 'w_IOError')
-        try:
-            dh = libssl_PEM_read_bio_DHparams(bio, None, None, None)
-        finally:
-            libssl_BIO_free(bio)
-        if not dh:
-            errno = get_saved_errno()
-            if errno != 0:
-                libssl_ERR_clear_error()
-                raise wrap_oserror(space, OSError(errno, ''))
-            else:
-                raise _ssl_seterror(space, None, 0)
-        try:
-            if libssl_SSL_CTX_set_tmp_dh(self.ctx, dh) == 0:
-                raise _ssl_seterror(space, None, 0)
-        finally:
-            libssl_DH_free(dh)
-
-    def load_verify_locations_w(self, space, w_cafile=None, w_capath=None,
-                                w_cadata=None):
-        if space.is_none(w_cafile):
-            cafile = None
-        else:
-            cafile = space.text_w(w_cafile)
-        if space.is_none(w_capath):
-            capath = None
-        else:
-            capath = space.text_w(w_capath)
-        if space.is_none(w_cadata):
-            cadata = None
-            ca_file_type = -1
-        else:
-            if not space.isinstance_w(w_cadata, space.w_unicode):
-                ca_file_type = SSL_FILETYPE_ASN1
-                cadata = space.bufferstr_w(w_cadata)
-            else:
-                ca_file_type = SSL_FILETYPE_PEM
-                w_uni = space.convert_arg_to_w_unicode(w_cadata)
-                if not w_uni.is_ascii():
-                    raise oefmt(space.w_TypeError,
-                                "cadata should be a ASCII string or a "
-                                "bytes-like object")
-                cadata = space.utf8_w(w_uni)
-
-        if cafile is None and capath is None and cadata is None:
-            raise oefmt(space.w_TypeError,
-                        "cafile and capath cannot be both omitted")
-        # load from cadata
-        if cadata is not None:
-            with rffi.scoped_nonmovingbuffer(cadata) as buf:
-                self._add_ca_certs(space, buf, len(cadata), ca_file_type)
-
-        # load cafile or capath
-        if cafile is not None or capath is not None:
-            ret = libssl_SSL_CTX_load_verify_locations(
-                self.ctx, cafile, capath)
-            if ret != 1:
-                errno = get_saved_errno()
-                if errno:
-                    libssl_ERR_clear_error()
-                    raise wrap_oserror(space, OSError(errno, ''),
-                                       exception_name = 'w_IOError')
-                else:
-                    raise _ssl_seterror(space, None, -1)
-
-    def _add_ca_certs(self, space, data, size, ca_file_type):
-        biobuf = libssl_BIO_new_mem_buf(data, size)
-        if not biobuf:
-            raise ssl_error(space, "Can't allocate buffer")
-        try:
-            store = libssl_SSL_CTX_get_cert_store(self.ctx)
-            loaded = 0
-            while True:
-                if ca_file_type == SSL_FILETYPE_ASN1:
-                    cert = libssl_d2i_X509_bio(
-                        biobuf, None)
-                else:
-                    cert = libssl_PEM_read_bio_X509(
-                        biobuf, None, None, None)
-                if not cert:
-                    break
-                try:
-                    r = libssl_X509_STORE_add_cert(store, cert)
-                finally:
-                    libssl_X509_free(cert)
-                if not r:
-                    err = libssl_ERR_peek_last_error()
-                    if (libssl_ERR_GET_LIB(err) == ERR_LIB_X509 and
-                        libssl_ERR_GET_REASON(err) ==
-                        X509_R_CERT_ALREADY_IN_HASH_TABLE):
-                        # cert already in hash table, not an error
-                        libssl_ERR_clear_error()
-                    else:
-                        break
-                loaded += 1
-
-            err = libssl_ERR_peek_last_error()
-            if (ca_file_type == SSL_FILETYPE_ASN1 and
-                loaded > 0 and
-                libssl_ERR_GET_LIB(err) == ERR_LIB_ASN1 and
-                libssl_ERR_GET_REASON(err) == ASN1_R_HEADER_TOO_LONG):
-                # EOF ASN1 file, not an error
-                libssl_ERR_clear_error()
-            elif (ca_file_type == SSL_FILETYPE_PEM and
-                  loaded > 0 and
-                  libssl_ERR_GET_LIB(err) == ERR_LIB_PEM and
-                  libssl_ERR_GET_REASON(err) == PEM_R_NO_START_LINE):
-                # EOF PEM file, not an error
-                libssl_ERR_clear_error()
-            else:
-                raise _ssl_seterror(space, None, 0)
-        finally:
-            libssl_BIO_free(biobuf)
-
-    def cert_store_stats_w(self, space):
-        store = libssl_SSL_CTX_get_cert_store(self.ctx)
-        x509 = 0
-        x509_ca = 0
-        crl = 0
-        objs = libssl_X509_STORE_get0_objects(store)
-        for i in range(libssl_sk_X509_OBJECT_num(objs)):
-            obj = libssl_sk_X509_OBJECT_value(objs, i)
-            obj_type = intmask(libssl_X509_OBJECT_get_type(obj))
-            if obj_type == X509_LU_X509:
-                x509 += 1
-                if libssl_X509_check_ca(
-                        libssl_X509_OBJECT_get0_X509(obj)):
-                    x509_ca += 1
-            elif obj_type == X509_LU_CRL:
-                crl += 1
-            else:
-                # Ignore X509_LU_FAIL, X509_LU_RETRY, X509_LU_PKEY.
-                # As far as I can tell they are internal states and never
-                # stored in a cert store
-                pass
-        w_result = space.newdict()
-        space.setitem(w_result,
-                      space.newtext('x509'), space.newint(x509))
-        space.setitem(w_result,
-                      space.newtext('x509_ca'), space.newint(x509_ca))
-        space.setitem(w_result,
-                      space.newtext('crl'), space.newint(crl))
-        return w_result
-
-    @unwrap_spec(protos='bufferstr')
-    def set_npn_protocols_w(self, space, protos):
-        if not HAS_NPN:
-            raise oefmt(space.w_NotImplementedError,
-                        "The NPN extension requires OpenSSL 1.0.1 or later.")
-
-        self.npn_protocols = SSLNpnProtocols(self.ctx, protos)
-
-    @unwrap_spec(protos='bufferstr')
-    def set_alpn_protocols_w(self, space, protos):
-        if not HAS_ALPN:
-            raise oefmt(space.w_NotImplementedError,
-                        "The ALPN extension requires OpenSSL 1.0.2 or later.")
-
-        self.alpn_protocols = SSLAlpnProtocols(self.ctx, protos)
-
-    def get_ca_certs_w(self, space, w_binary_form=None):
-        if w_binary_form and space.is_true(w_binary_form):
-            binary_mode = True
-        else:
-            binary_mode = False
-        rlist = []
-        store = libssl_SSL_CTX_get_cert_store(self.ctx)
-        objs = libssl_X509_STORE_get0_objects(store)
-        for i in range(libssl_sk_X509_OBJECT_num(objs)):
-            obj = libssl_sk_X509_OBJECT_value(objs, i)
-            if intmask(libssl_X509_OBJECT_get_type(obj)) != X509_LU_X509:
-                # not a x509 cert
-                continue
-            # CA for any purpose
-            cert = libssl_X509_OBJECT_get0_X509(obj)
-            if not libssl_X509_check_ca(cert):
-                continue
-            if binary_mode:
-                rlist.append(_certificate_to_der(space, cert))
-            else:
-                rlist.append(_decode_certificate(space, cert))
-        return space.newlist(rlist)
-
-    @unwrap_spec(name='text')
-    def set_ecdh_curve_w(self, space, name):
-        nid = libssl_OBJ_sn2nid(name)
-        if nid == 0:
-            raise oefmt(space.w_ValueError,
-                        "unknown elliptic curve name '%s'", name)
-        key = libssl_EC_KEY_new_by_curve_name(nid)
-        if not key:
-            raise _ssl_seterror(space, None, 0)
-        try:
-            libssl_SSL_CTX_set_tmp_ecdh(self.ctx, key)
-        finally:
-            libssl_EC_KEY_free(key)
-
-    def set_servername_callback_w(self, space, w_callback):
-        if space.is_none(w_callback):
-            libssl_SSL_CTX_set_tlsext_servername_callback(
-                self.ctx, lltype.nullptr(servername_cb.TO))
-            self.servername_callback = None
-            return
-        if not space.is_true(space.callable(w_callback)):
-            raise oefmt(space.w_TypeError, "not a callable object")
-        callback_struct = ServernameCallback()
-        callback_struct.space = space
-        callback_struct.w_ctx = self
-        callback_struct.w_set_hostname = w_callback
-        self.servername_callback = callback_struct
-        index = compute_unique_id(self)
-        SERVERNAME_CALLBACKS.set(index, callback_struct)
-        libssl_SSL_CTX_set_tlsext_servername_callback(
-            self.ctx, _servername_callback)
-        libssl_SSL_CTX_set_tlsext_servername_arg(self.ctx,
-                                                 rffi.cast(rffi.VOIDP, index))
-
-_SSLContext.typedef = TypeDef(
-    "_ssl._SSLContext",
-    __new__=interp2app(_SSLContext.descr_new),
-    _wrap_socket=interp2app(_SSLContext.descr_wrap_socket),
-    set_ciphers=interp2app(_SSLContext.descr_set_ciphers),
-    cert_store_stats=interp2app(_SSLContext.cert_store_stats_w),
-    load_cert_chain=interp2app(_SSLContext.load_cert_chain_w),
-    load_dh_params=interp2app(_SSLContext.load_dh_params_w),
-    load_verify_locations=interp2app(_SSLContext.load_verify_locations_w),
-    session_stats = interp2app(_SSLContext.session_stats_w),
-    set_default_verify_paths=interp2app(_SSLContext.descr_set_default_verify_paths),
-    _set_npn_protocols=interp2app(_SSLContext.set_npn_protocols_w),
-    _set_alpn_protocols=interp2app(_SSLContext.set_alpn_protocols_w),
-    get_ca_certs=interp2app(_SSLContext.get_ca_certs_w),
-    set_ecdh_curve=interp2app(_SSLContext.set_ecdh_curve_w),
-    set_servername_callback=interp2app(_SSLContext.set_servername_callback_w),
-
-    options=GetSetProperty(_SSLContext.descr_get_options,
-                           _SSLContext.descr_set_options),
-    verify_mode=GetSetProperty(_SSLContext.descr_get_verify_mode,
-                               _SSLContext.descr_set_verify_mode),
-    verify_flags=GetSetProperty(_SSLContext.descr_get_verify_flags,
-                                _SSLContext.descr_set_verify_flags),
-    check_hostname=GetSetProperty(_SSLContext.descr_get_check_hostname,
-                                  _SSLContext.descr_set_check_hostname),
-)
-
-
-def _asn1obj2py(space, obj):
-    nid = libssl_OBJ_obj2nid(obj)
-    if nid == NID_undef:
-        raise oefmt(space.w_ValueError, "Unknown object")
-    with rffi.scoped_alloc_buffer(100) as buf:
-        buflen = libssl_OBJ_obj2txt(buf.raw, 100, obj, 1)
-        if buflen < 0:
-            raise _ssl_seterror(space, None, 0)
-        if buflen:
-            w_buf = space.newtext(buf.str(buflen))
-        else:
-            w_buf = space.w_None
-    w_sn = space.newtext(rffi.charp2str(libssl_OBJ_nid2sn(nid)))
-    w_ln = space.newtext(rffi.charp2str(libssl_OBJ_nid2ln(nid)))
-    return space.newtuple([space.newint(nid), w_sn, w_ln, w_buf])
-
-
- at unwrap_spec(txt='bytes', name=bool)
-def txt2obj(space, txt, name=False):
-    obj = libssl_OBJ_txt2obj(txt, not name)
-    if not obj:
-        raise oefmt(space.w_ValueError, "unknown object '%s'", txt)
-    try:
-        w_result = _asn1obj2py(space, obj)
-    finally:
-        libssl_ASN1_OBJECT_free(obj)
-    return w_result
-
-
- at unwrap_spec(nid=int)
-def nid2obj(space, nid):
-    if nid < NID_undef:
-        raise oefmt(space.w_ValueError, "NID must be positive")
-    obj = libssl_OBJ_nid2obj(nid)
-    if not obj:
-        raise oefmt(space.w_ValueError, "unknown NID %d", nid)
-    try:
-        w_result = _asn1obj2py(space, obj)
-    finally:
-        libssl_ASN1_OBJECT_free(obj)
-    return w_result
-
-
-def w_convert_path(space, path):
-    if not path:
-        return space.w_None
-    else:
-        return space.newbytes(rffi.charp2str(path))
-
-def get_default_verify_paths(space):
-    return space.newtuple([
-        w_convert_path(space, libssl_X509_get_default_cert_file_env()),
-        w_convert_path(space, libssl_X509_get_default_cert_file()),
-        w_convert_path(space, libssl_X509_get_default_cert_dir_env()),
-        w_convert_path(space, libssl_X509_get_default_cert_dir()),
-        ])
diff --git a/pypy/module/_ssl/interp_win32.py b/pypy/module/_ssl/interp_win32.py
deleted file mode 100644
--- a/pypy/module/_ssl/interp_win32.py
+++ /dev/null
@@ -1,180 +0,0 @@
-from rpython.rlib import rwin32
-from rpython.rtyper.lltypesystem import rffi, lltype
-from rpython.rtyper.tool import rffi_platform
-from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from pypy.interpreter.gateway import unwrap_spec
-from pypy.interpreter.error import wrap_windowserror
-from rpython.rlib.rarithmetic import intmask
-eci = ExternalCompilationInfo(
-    includes = ['windows.h', 'wincrypt.h'],
-    libraries = ['crypt32'],
-)
-
-class CConfig:
-    _compilation_info_ = eci
-
-    X509_ASN_ENCODING = rffi_platform.ConstantInteger('X509_ASN_ENCODING')
-    PKCS_7_ASN_ENCODING = rffi_platform.ConstantInteger('PKCS_7_ASN_ENCODING')
-    CERT_FIND_PROP_ONLY_ENHKEY_USAGE_FLAG = rffi_platform.ConstantInteger('CERT_FIND_PROP_ONLY_ENHKEY_USAGE_FLAG')
-    CERT_FIND_EXT_ONLY_ENHKEY_USAGE_FLAG = rffi_platform.ConstantInteger('CERT_FIND_EXT_ONLY_ENHKEY_USAGE_FLAG')
-    CRYPT_E_NOT_FOUND = rffi_platform.ConstantInteger('CRYPT_E_NOT_FOUND')
-
-    CERT_ENHKEY_USAGE = rffi_platform.Struct(
-        'CERT_ENHKEY_USAGE', [('cUsageIdentifier', rwin32.DWORD),
-                              ('rgpszUsageIdentifier', rffi.CCHARPP)])
-    CERT_CONTEXT = rffi_platform.Struct(
-        'CERT_CONTEXT', [('pbCertEncoded', rffi.CCHARP),
-                         ('cbCertEncoded', rwin32.DWORD),
-                         ('dwCertEncodingType', rwin32.DWORD)])
-    CRL_CONTEXT = rffi_platform.Struct(
-        'CRL_CONTEXT', [('pbCrlEncoded', rffi.CCHARP),
-                        ('cbCrlEncoded', rwin32.DWORD),
-                        ('dwCertEncodingType', rwin32.DWORD)])
-
-for k, v in rffi_platform.configure(CConfig).items():
-    globals()[k] = v
-
-PCERT_ENHKEY_USAGE = lltype.Ptr(CERT_ENHKEY_USAGE)
-PCCERT_CONTEXT = lltype.Ptr(CERT_CONTEXT)
-PCCRL_CONTEXT = lltype.Ptr(CRL_CONTEXT)
-
-def external(name, argtypes, restype, **kw):
-    kw['compilation_info'] = eci
-    kw['calling_conv'] = 'win'
-    kw['save_err'] = rffi.RFFI_SAVE_LASTERROR
-    return rffi.llexternal(
-        name, argtypes, restype, **kw)
-
-CertOpenSystemStore = external(
-    'CertOpenSystemStoreA', [rffi.VOIDP, rffi.CCHARP], rwin32.HANDLE)
-CertCloseStore = external(
-    'CertCloseStore', [rwin32.HANDLE, rwin32.DWORD], rwin32.BOOL)
-CertGetEnhancedKeyUsage = external(
-    'CertGetEnhancedKeyUsage',
-    [PCCERT_CONTEXT, rwin32.DWORD, PCERT_ENHKEY_USAGE, rwin32.LPDWORD],
-    rwin32.BOOL)
-CertEnumCertificatesInStore = external(
-    'CertEnumCertificatesInStore',
-    [rwin32.HANDLE, PCCERT_CONTEXT], PCCERT_CONTEXT)
-CertFreeCertificateContext = external(
-    'CertFreeCertificateContext', [PCCERT_CONTEXT], rwin32.BOOL)
-CertEnumCRLsInStore = external(
-    'CertEnumCRLsInStore',
-    [rwin32.HANDLE, PCCRL_CONTEXT], PCCRL_CONTEXT)
-CertFreeCRLContext = external(
-    'CertFreeCRLContext', [PCCRL_CONTEXT], rwin32.BOOL)
-
-def w_certEncodingType(space, encodingType):
-    if encodingType == X509_ASN_ENCODING:
-        return space.newtext("x509_asn")
-    elif encodingType == PKCS_7_ASN_ENCODING:
-        return space.newtext("pkcs_7_asn")
-    else:
-        return space.newint(encodingType)
-
-def w_parseKeyUsage(space, pCertCtx, flags):
-    with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as size_ptr:
-        if not CertGetEnhancedKeyUsage(pCertCtx, flags,
-                             lltype.nullptr(CERT_ENHKEY_USAGE), size_ptr):
-            last_error = rwin32.lastSavedWindowsError()
-            if last_error.winerror == CRYPT_E_NOT_FOUND:
-                return space.w_True
-            raise wrap_windowserror(space, last_error)
-
-        size = intmask(size_ptr[0])
-        with lltype.scoped_alloc(rffi.CCHARP.TO, size) as buf:
-            usage = rffi.cast(PCERT_ENHKEY_USAGE, buf)
-            # Now get the actual enhanced usage property
-            if not CertGetEnhancedKeyUsage(pCertCtx, flags, usage, size_ptr):
-                last_error= rwin32.lastSavedWindowsError()
-                if last_error.winerror == CRYPT_E_NOT_FOUND:
-                    return space.w_True
-                raise wrap_windowserror(space, last_error)
-
-            result_w = [None] * usage.c_cUsageIdentifier
-            for i in range(usage.c_cUsageIdentifier):
-                if not usage.c_rgpszUsageIdentifier[i]:
-                    continue
-                result_w[i] = space.newtext(rffi.charp2str(
-                    usage.c_rgpszUsageIdentifier[i]))
-            return space.newset(result_w)
-
- at unwrap_spec(store_name='text')
-def enum_certificates_w(space, store_name):
-    """enum_certificates(store_name) -> []
-
-    Retrieve certificates from Windows' cert store. store_name may be one of
-    'CA', 'ROOT' or 'MY'. The system may provide more cert storages, too.
-    The function returns a list of (bytes, encoding_type, trust) tuples. The
-    encoding_type flag can be interpreted with X509_ASN_ENCODING or
-    PKCS_7_ASN_ENCODING. The trust setting is either a set of OIDs or the
-    boolean True."""
-
-    result_w = []
-    pCertCtx = lltype.nullptr(CERT_CONTEXT)
-    hStore = CertOpenSystemStore(None, store_name)
-    if not hStore:
-        raise wrap_windowserror(space, rwin32.lastSavedWindowsError())
-    try:
-        while True:
-            pCertCtx = CertEnumCertificatesInStore(hStore, pCertCtx)
-            if not pCertCtx:
-                break
-            w_cert = space.newbytes(
-                rffi.charpsize2str(pCertCtx.c_pbCertEncoded,
-                                   intmask(pCertCtx.c_cbCertEncoded)))
-            w_enc = w_certEncodingType(space, pCertCtx.c_dwCertEncodingType)
-            w_keyusage = w_parseKeyUsage(
-                space, pCertCtx, CERT_FIND_PROP_ONLY_ENHKEY_USAGE_FLAG)
-            if space.is_w(w_keyusage, space.w_True):
-                w_keyusage = w_parseKeyUsage(
-                    space, pCertCtx, CERT_FIND_EXT_ONLY_ENHKEY_USAGE_FLAG)
-            result_w.append(space.newtuple([w_cert, w_enc, w_keyusage]))
-    except:
-        raise
-    finally:
-        if pCertCtx:
-            # loop ended with an error, need to clean up context manually
-            CertFreeCertificateContext(pCertCtx)
-        if not CertCloseStore(hStore, 0):
-            # This error case might shadow another exception.
-            raise wrap_windowserror(space, rwin32.lastSavedWindowsError())
-
-    return space.newlist(result_w)

From pypy.commits at gmail.com  Tue Oct 22 14:37:45 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 22 Oct 2019 11:37:45 -0700 (PDT)
Subject: [pypy-commit] pypy default: issue 2389: (take 2) redo faulty logic
 by copying runicode.str_decode_utf8_impl
Message-ID: <5daf4c79.1c69fb81.3c651.5eba@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r97836:c4b55d31320d
Date: 2019-10-22 20:20 +0300
http://bitbucket.org/pypy/pypy/changeset/c4b55d31320d/

Log:	issue 2389: (take 2) redo faulty logic by copying
	runicode.str_decode_utf8_impl

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -7,6 +7,7 @@
 from rpython.rlib.rarithmetic import r_uint, intmask
 from rpython.rtyper.lltypesystem import rffi
 from pypy.module.unicodedata.interp_ucd import unicodedb
+from rpython.rlib import runicode
 
 @specialize.memo()
 def decode_error_handler(space):
@@ -222,14 +223,12 @@
 
 if sys.platform == 'win32':
     def utf8_encode_mbcs(s, errors, errorhandler):
-        from rpython.rlib import runicode
         s = s.decode('utf-8')
         slen = len(s)
         res = runicode.unicode_encode_mbcs(s, slen, errors, errorhandler)
         return res
 
     def str_decode_mbcs(s, errors, final, errorhandler):
-        from rpython.rlib import runicode
         slen = len(s)
         res, size = runicode.str_decode_mbcs(s, slen, final=final, errors=errors,
                                            errorhandler=errorhandler)
@@ -240,134 +239,156 @@
     valid so we're trying to either raise or pack stuff with error handler.
     The key difference is that this is call_may_force
     """
-    slen = len(s)
-    res = StringBuilder(slen)
+
+    # Adapted from str_decode_utf_8_impl but with a StringBuilder, not a
+    # UnicodeBuilder
+    size = len(s)
+    result = StringBuilder(size)
     pos = 0
-    end = len(s)
-    suppressing = False # we are in a chain of "bad" unicode, only emit one fix
-    while pos < end:
+    while pos < size:
         ordch1 = ord(s[pos])
         # fast path for ASCII
-        if ordch1 <= 0x7F:
+        if ordch1 < 0x80:
+            result.append(chr(ordch1))
             pos += 1
-            res.append(chr(ordch1))
-            suppressing = False
             continue
 
-        if ordch1 <= 0xC1:
-            r, pos = errorhandler(errors, "utf8", "invalid start byte",
-                    s, pos, pos + 1)
-            if not suppressing:
-                res.append(r)
-            continue
+        n = ord(runicode._utf8_code_length[ordch1 - 0x80])
+        if pos + n > size:
+            if not final:
+                break
+            # argh, this obscure block of code is mostly a copy of
+            # what follows :-(
+            charsleft = size - pos - 1 # either 0, 1, 2
+            # note: when we get the 'unexpected end of data' we need
+            # to care about the pos returned; it can be lower than size,
+            # in case we need to continue running this loop
+            if not charsleft:
+                # there's only the start byte and nothing else
+                r, pos = errorhandler(errors, 'utf8',
+                                      'unexpected end of data',
+                                      s, pos, pos+1)
+                result.append(r)
+                continue
+            ordch2 = ord(s[pos+1])
+            if n == 3:
+                # 3-bytes seq with only a continuation byte
+                if rutf8._invalid_byte_2_of_3(ordch1, ordch2, True):
+                    # second byte invalid, take the first and continue
+                    r, pos = errorhandler(errors, 'utf8',
+                                          'invalid continuation byte',
+                                          s, pos, pos+1)
+                    result.append(r)
+                    continue
+                else:
+                    # second byte valid, but third byte missing
+                    r, pos = errorhandler(errors, 'utf8',
+                                      'unexpected end of data',
+                                      s, pos, pos+2)
+                    result.append(r)
+                    continue
+            elif n == 4:
+                # 4-bytes seq with 1 or 2 continuation bytes
+                if rutf8._invalid_byte_2_of_4(ordch1, ordch2):
+                    # second byte invalid, take the first and continue
+                    r, pos = errorhandler(errors, 'utf8',
+                                          'invalid continuation byte',
+                                          s, pos, pos+1)
+                    result.append(r)
+                    continue
+                elif charsleft == 2 and rutf8._invalid_byte_3_of_4(ord(s[pos+2])):
+                    # third byte invalid, take the first two and continue
+                    r, pos = errorhandler(errors, 'utf8',
+                                          'invalid continuation byte',
+                                          s, pos, pos+2)
+                    result.append(r)
+                    continue
+                else:
+                    # there's only 1 or 2 valid cb, but the others are missing
+                    r, pos = errorhandler(errors, 'utf8',
+                                      'unexpected end of data',
+                                      s, pos, pos+charsleft+1)
+                    result.append(r)
+                    continue
+            raise AssertionError("unreachable")
 
-        pos += 1
+        if n == 0:
+            r, pos = errorhandler(errors, 'utf8',
+                                  'invalid start byte',
+                                  s, pos, pos+1)
+            result.append(r)
 
-        if ordch1 <= 0xDF:
-            if pos >= end:
-                if not final:
-                    pos -= 1
-                    break
-                r, pos = errorhandler(errors, "utf8", "unexpected end of data",
-                    s, pos - 1, pos)
-                if not suppressing:
-                    res.append(r)
-                continue
-            ordch2 = ord(s[pos])
+        elif n == 1:
+            assert 0, "ascii should have gone through the fast path"
 
+        elif n == 2:
+            ordch2 = ord(s[pos+1])
             if rutf8._invalid_byte_2_of_2(ordch2):
-                r, pos = errorhandler(errors, "utf8", "invalid continuation byte",
-                    s, pos - 1, pos)
-                if not suppressing:
-                    res.append(r)
+                r, pos = errorhandler(errors, 'utf8',
+                                      'invalid continuation byte',
+                                      s, pos, pos+1)
+                result.append(r)
                 continue
             # 110yyyyy 10zzzzzz -> 00000000 00000yyy yyzzzzzz
-            pos += 1
-            res.append(chr(ordch1))
-            res.append(chr(ordch2))
-            continue
+            result.append(chr(ordch1))
+            result.append(chr(ordch2))
+            pos += 2
 
-        if ordch1 <= 0xEF:
-            if (pos + 2) > end:
-                if not final:
-                    pos -= 1
-                    break
-                r, pos = errorhandler(errors, "utf8", "unexpected end of data",
-                    s, pos - 1, pos)
-                res.append(r)
-                suppressing = True
-                continue
-            ordch2 = ord(s[pos])
-            ordch3 = ord(s[pos + 1])
+        elif n == 3:
+            ordch2 = ord(s[pos + 1])
+            ordch3 = ord(s[pos + 2])
 
             if rutf8._invalid_byte_2_of_3(ordch1, ordch2, True):
-                r, pos = errorhandler(errors, "utf8", "invalid continuation byte",
-                    s, pos - 1, pos)
-                if not suppressing:
-                    res.append(r)
+                r, pos = errorhandler(errors, 'utf8',
+                                      'invalid continuation byte',
+                                      s, pos, pos + 1)
+                result.append(r)
                 continue
             elif rutf8._invalid_byte_3_of_3(ordch3):
-                r, pos = errorhandler(errors, "utf8", "invalid continuation byte",
-                    s, pos - 1, pos + 1)
-                if not suppressing:
-                    res.append(r)
+                r, pos = errorhandler(errors, 'utf8',
+                                      "invalid continuation byte",
+                                      s, pos, pos + 2)
+                result.append(r)
                 continue
-            pos += 2
 
             # 1110xxxx 10yyyyyy 10zzzzzz -> 00000000 xxxxyyyy yyzzzzzz
-            res.append(chr(ordch1))
-            res.append(chr(ordch2))
-            res.append(chr(ordch3))
-            suppressing = False
-            continue
+            result.append(chr(ordch1))
+            result.append(chr(ordch2))
+            result.append(chr(ordch3))
+            pos += 3
 
-        if ordch1 <= 0xF4:
-            if (pos + 3) > end:
-                if not final:
-                    pos -= 1
-                    break
-                r, pos = errorhandler(errors, "utf8", "unexpected end of data",
-                    s, pos - 1, pos)
-                res.append(r)
-                suppressing = True
-                continue
-            ordch2 = ord(s[pos])
-            ordch3 = ord(s[pos + 1])
-            ordch4 = ord(s[pos + 2])
+        elif n == 4:
+            ordch2 = ord(s[pos + 1])
+            ordch3 = ord(s[pos + 2])
+            ordch4 = ord(s[pos + 3])
 
             if rutf8._invalid_byte_2_of_4(ordch1, ordch2):
-                r, pos = errorhandler(errors, "utf8", "invalid continuation byte",
-                    s, pos - 1, pos)
-                if not suppressing:
-                    res.append(r)
+                r, pos = errorhandler(errors, 'utf8',
+                                      'invalid continuation byte',
+                                      s, pos, pos + 1)
+                result.append(r)
                 continue
             elif rutf8._invalid_byte_3_of_4(ordch3):
-                r, pos = errorhandler(errors, "utf8", "invalid continuation byte",
-                    s, pos - 1, pos + 1)
-                res.append(r)
+                r, pos = errorhandler(errors, 'utf8',
+                                     'invalid continuation byte',
+                                     s, pos, pos + 2)
+                result.append(r)
                 continue
             elif rutf8._invalid_byte_4_of_4(ordch4):
-                r, pos = errorhandler(errors, "utf8", "invalid continuation byte",
-                    s, pos - 1, pos + 2)
-                if not suppressing:
-                    res.append(r)
+                r, pos = errorhandler(errors, 'utf8',
+                                      'invalid continuation byte',
+                                      s, pos, pos + 3)
+                result.append(r)
                 continue
 
-            pos += 3
             # 11110www 10xxxxxx 10yyyyyy 10zzzzzz -> 000wwwxx xxxxyyyy yyzzzzzz
-            res.append(chr(ordch1))
-            res.append(chr(ordch2))
-            res.append(chr(ordch3))
-            res.append(chr(ordch4))
-            suppressing = False
-            continue
+            result.append(chr(ordch1))
+            result.append(chr(ordch2))
+            result.append(chr(ordch3))
+            result.append(chr(ordch4))
+            pos += 4
 
-        r, pos = errorhandler(errors, "utf8", "invalid start byte",
-                s, pos - 1, pos)
-        if not suppressing:
-            res.append(r)
-
-    r = res.build()
+    r = result.build()
     return r, pos, rutf8.check_utf8(r, True)
 
 hexdigits = "0123456789ABCDEFabcdef"
diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -938,3 +938,17 @@
         errors = []
         assert sin.encode("iso-8859-15", "test.record") == "\xac\xa4"
         assert errors == [u'\u1234\u1234', u'\u8000']
+
+    def test_last_byte_handler(self):
+        # issue bb-2389
+        import _codecs
+        _codecs.register_error('custom_replace', lambda exc: (u'\ufffd', exc.start+1))
+        for s, res in ((b"WORD\xe3\xab",
+                            (u'WORD\ufffd\ufffd', u'WORD\ufffd')),
+                       (b"\xef\xbb\xbfWORD\xe3\xabWORD2",
+                            (u'\ufeffWORD\ufffd\ufffdWORD2',
+                             u'\ufeffWORD\ufffdWORD2'))):
+            r = s.decode('utf8', 'replace')
+            assert r == res[1]
+            r = s.decode('utf8', 'custom_replace')
+            assert r == res[0]

From pypy.commits at gmail.com  Wed Oct 23 07:32:51 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 23 Oct 2019 04:32:51 -0700 (PDT)
Subject: [pypy-commit] pypy default: (cfbolz,
 mattip around) print instructions how to rebuild the _pypy_openssl
 cffi module
Message-ID: <5db03a63.1c69fb81.8862d.21fc@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r97838:cdba165e6a48
Date: 2019-10-23 13:31 +0200
http://bitbucket.org/pypy/pypy/changeset/cdba165e6a48/

Log:	(cfbolz, mattip around) print instructions how to rebuild the
	_pypy_openssl cffi module

diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -2,8 +2,19 @@
 import time
 import thread as _thread
 import weakref
-from _pypy_openssl import ffi
-from _pypy_openssl import lib
+
+try:
+    from _pypy_openssl import ffi
+    from _pypy_openssl import lib
+except ImportError:
+    import os
+    print("The _ssl cffi module either doesn't exist or is incompatible with your machine's shared libraries.")
+    print("If you have a compiler installed, you can try to rebuild it by running:")
+    print("cd %s" % os.path.abspath(os.path.dirname(os.path.dirname(__file__))))
+    print("%s _ssl_build.py" % sys.executable)
+    print()
+    raise
+
 from _cffi_ssl._stdssl.certificate import (_test_decode_cert,
     _decode_certificate, _certificate_to_der)
 from _cffi_ssl._stdssl.utility import (_str_with_len, _bytes_with_len,

From pypy.commits at gmail.com  Wed Oct 23 09:36:07 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 23 Oct 2019 06:36:07 -0700 (PDT)
Subject: [pypy-commit] pypy default: (cfbolz,
 arigo around) attach error to exception
Message-ID: <5db05747.1c69fb81.eea3f.1e45@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r97839:b6cf80ef9029
Date: 2019-10-23 15:35 +0200
http://bitbucket.org/pypy/pypy/changeset/b6cf80ef9029/

Log:	(cfbolz, arigo around) attach error to exception

diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -6,14 +6,13 @@
 try:
     from _pypy_openssl import ffi
     from _pypy_openssl import lib
-except ImportError:
+except ImportError as e:
     import os
-    print("The _ssl cffi module either doesn't exist or is incompatible with your machine's shared libraries.")
-    print("If you have a compiler installed, you can try to rebuild it by running:")
-    print("cd %s" % os.path.abspath(os.path.dirname(os.path.dirname(__file__))))
-    print("%s _ssl_build.py" % sys.executable)
-    print()
-    raise
+    msg = "\n\nThe _ssl cffi module either doesn't exist or is incompatible with your machine's shared libraries.\n" + \
+          "If you have a compiler installed, you can try to rebuild it by running:\n" + \
+          "cd %s\n" % os.path.abspath(os.path.dirname(os.path.dirname(__file__))) + \
+          "%s _ssl_build.py\n" % sys.executable
+    raise ImportError(str(e) + msg)
 
 from _cffi_ssl._stdssl.certificate import (_test_decode_cert,
     _decode_certificate, _certificate_to_der)

From pypy.commits at gmail.com  Thu Oct 24 01:50:20 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 23 Oct 2019 22:50:20 -0700 (PDT)
Subject: [pypy-commit] pypy expose-jsonmap: expose a way to create a dict
 structure from applevel, and instantiate it. can
Message-ID: <5db13b9c.1c69fb81.397fc.5c73@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: expose-jsonmap
Changeset: r97840:8504b741f563
Date: 2019-10-24 07:49 +0200
http://bitbucket.org/pypy/pypy/changeset/8504b741f563/

Log:	expose a way to create a dict structure from applevel, and
	instantiate it. can be used in csv module and other places.

diff --git a/pypy/module/__pypy__/interp_dictstructure.py b/pypy/module/__pypy__/interp_dictstructure.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__pypy__/interp_dictstructure.py
@@ -0,0 +1,26 @@
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.error import oefmt
+from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.typedef import TypeDef
+
+def make_terminator(space):
+    from pypy.module._pypyjson.interp_decoder import Terminator
+    return Terminator(space)
+
+def newdictstructure(space, w_list=None):
+    from pypy.objspace.std.unicodeobject import W_UnicodeObject
+    m = terminator = space.fromcache(make_terminator) # a different one than the one _pypyjson uses
+    if w_list is None:
+        return m
+    for w_x in space.listview(w_list):
+        if type(w_x) is not W_UnicodeObject:
+            raise oefmt(space.w_TypeError, "expected unicode, got %T", w_x)
+        u = space.utf8_w(w_x)
+        m = m.get_next(w_x, u, 0, len(u), terminator)
+        if not m.is_state_useful():
+            m.mark_useful(terminator)  # XXX too aggressive?
+    return m
+
+
+
+
diff --git a/pypy/module/__pypy__/moduledef.py b/pypy/module/__pypy__/moduledef.py
--- a/pypy/module/__pypy__/moduledef.py
+++ b/pypy/module/__pypy__/moduledef.py
@@ -118,6 +118,8 @@
         'stack_almost_full'         : 'interp_magic.stack_almost_full',
         'pyos_inputhook'            : 'interp_magic.pyos_inputhook',
         'newmemoryview'             : 'interp_buffer.newmemoryview',
+
+        'newdictstructure'          : 'interp_dictstructure.newdictstructure',
     }
     if sys.platform == 'win32':
         interpleveldefs['get_console_cp'] = 'interp_magic.get_console_cp'
diff --git a/pypy/module/__pypy__/test/test_dictstructure.py b/pypy/module/__pypy__/test/test_dictstructure.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__pypy__/test/test_dictstructure.py
@@ -0,0 +1,26 @@
+
+class AppTestDictStructure(object):
+    spaceconfig = dict(usemodules=['__pypy__', '_pypyjson'])
+
+    def test_simple(self):
+        from __pypy__ import newdictstructure, strategy
+        m = newdictstructure([u"a", u"b", u"c"])
+        assert m.last_key == u"c"
+        assert m.previous.last_key == u"b"
+        m2 = newdictstructure([u"a", u"b", u"c"])
+        assert m2 is m
+        d = m.instantiate_dict([1, 2, 3])
+        assert d == {u"a": 1, u"b": 2, u"c": 3}
+        assert strategy(d) == "JsonDictStrategy"
+
+        with raises(ValueError):
+            m.instantiate_dict([5])
+
+        with raises(TypeError):
+            newdictstructure([b"a"])
+
+
+    def test_repr(self):
+        from __pypy__ import newdictstructure, strategy
+        m = newdictstructure([u"a"])
+        assert repr(m) == "<DictStructure [u'a']>"
diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -9,6 +9,9 @@
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.module._pypyjson import simd
 
+from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.typedef import TypeDef, GetSetProperty
+
 OVF_DIGITS = len(str(sys.maxint))
 
 def is_whitespace(ch):
@@ -692,7 +695,7 @@
         return True
 
 
-class MapBase(object):
+class MapBase(W_Root):
     """ A map implementation to speed up parsing of json dicts, and to
     represent the resulting dicts more compactly and make access faster. """
 
@@ -866,6 +869,30 @@
         reading from values_w."""
         raise NotImplementedError("abstract base")
 
+    # ____________________________________________________________
+    # exposed methods
+
+    def descr_instantiate_dict(self, space, w_l):
+        from pypy.objspace.std.jsondict import from_values_and_jsonmap
+        from pypy.objspace.std.jsondict import devolve_jsonmap_dict
+        l_w = space.listview(w_l)
+        if not isinstance(self, JSONMap):
+            return space.newdict()
+        keys_w = self.get_keys_in_order()
+        if len(l_w) != len(keys_w):
+            raise oefmt(space.w_ValueError, "expected %s values, got %s", str(len(keys_w)), str(len(l_w)))
+        w_dict = from_values_and_jsonmap(self.space, l_w[:], self)
+        if self.is_state_blocked():
+            devolve_jsonmap_dict(w_dict)
+        return w_dict
+
+    def descr_repr(self, space):
+        return space.newtext("<DictStructure [%s]>" % ", ".join([space.text_w(space.repr(w_key)) for w_key in self.get_keys_in_order()]))
+
+
+    # ____________________________________________________________
+    # graphviz
+
     def _all_dot(self, output):
         identity = objectmodel.compute_unique_id(self)
         output.append('%s [shape=box%s];' % (identity, self._get_dot_text()))
@@ -1166,3 +1193,18 @@
     finally:
         decoder.close()
 
+
+
+def get_last_key(w_obj, space):
+    return w_obj.w_key if isinstance(w_obj, JSONMap) else space.w_None
+
+def get_previous(w_obj, space):
+    return w_obj.prev if isinstance(w_obj, JSONMap) else space.w_None
+
+MapBase.typedef = TypeDef("DictStructure",
+    instantiate_dict = interp2app(MapBase.descr_instantiate_dict),
+    __repr__ = interp2app(MapBase.descr_repr),
+    last_key = GetSetProperty(get_last_key, name="last_key"),
+    previous = GetSetProperty(get_previous, name="previous"),
+)
+MapBase.typedef.acceptable_as_base_class = False

From pypy.commits at gmail.com  Thu Oct 24 02:13:54 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 23 Oct 2019 23:13:54 -0700 (PDT)
Subject: [pypy-commit] pypy record-known-result: missing canrun=True
Message-ID: <5db14122.1c69fb81.86c56.66bb@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: record-known-result
Changeset: r97841:010c6d000930
Date: 2019-10-18 13:52 +0200
http://bitbucket.org/pypy/pypy/changeset/010c6d000930/

Log:	missing canrun=True

diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -473,7 +473,7 @@
     'gc_get_type_info_group': LLOp(sideeffects=False),
     'll_read_timestamp': LLOp(revdb_protect=True, canrun=True),
     'll_get_timestamp_unit': LLOp(revdb_protect=True, canrun=True),
-    'jit_record_known_result': LLOp(),
+    'jit_record_known_result': LLOp(canrun=True),
 
     # __________ GC operations __________
 

From pypy.commits at gmail.com  Thu Oct 24 02:23:20 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 23 Oct 2019 23:23:20 -0700 (PDT)
Subject: [pypy-commit] pypy record-known-result: this one turned out to be
 necessary
Message-ID: <5db14358.1c69fb81.44c2e.af29@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: record-known-result
Changeset: r97842:83b888a641b6
Date: 2019-10-24 08:22 +0200
http://bitbucket.org/pypy/pypy/changeset/83b888a641b6/

Log:	this one turned out to be necessary

diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -367,6 +367,9 @@
             # There is no call to codepoint_position_at_index here! It's
             # deduced as the reverse of _codepoint_index_at_byte_position.
             guard_no_exception(descr=...)
+            i111 = int_ge(i110, i44)
+            guard_false(i111, descr=...)
+
             i112 = call_i(ConstClass(next_codepoint_pos_dont_look_inside), p47, i105, descr=...)
             i114 = int_add(i110, 1)
             i115 = int_sub(i112, i105)

From pypy.commits at gmail.com  Thu Oct 24 02:27:11 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 23 Oct 2019 23:27:11 -0700 (PDT)
Subject: [pypy-commit] pypy record-known-result: document branch
Message-ID: <5db1443f.1c69fb81.34b28.87c2@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: record-known-result
Changeset: r97843:ed83b0a69574
Date: 2019-10-24 08:26 +0200
http://bitbucket.org/pypy/pypy/changeset/ed83b0a69574/

Log:	document branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -9,3 +9,9 @@
 
 Fix segfault when calling descr-methods with no arguments
 
+
+.. branch: record-known-result
+
+Improve reasoning of the JIT about utf-8 index manipulation. Add a generic
+framework to give hints that help the JIT reason about elidable calls and
+more generically, other invariants of the code.

From pypy.commits at gmail.com  Thu Oct 24 07:54:31 2019
From: pypy.commits at gmail.com (arigo)
Date: Thu, 24 Oct 2019 04:54:31 -0700 (PDT)
Subject: [pypy-commit] pypy record-known-result: Test and fix annotation for
 record_exact_value()
Message-ID: <5db190f7.1c69fb81.bed4a.9c7e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: record-known-result
Changeset: r97844:7e2f7fad5601
Date: 2019-10-24 13:53 +0200
http://bitbucket.org/pypy/pypy/changeset/7e2f7fad5601/

Log:	Test and fix annotation for record_exact_value()

diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -1213,7 +1213,7 @@
         func(*args)
     return _jit_record_known_result(result, func, *args)
 
-class ConditionalCallEntry(ExtRegistryEntry):
+class Entry(ExtRegistryEntry):
     _about_ = _jit_record_known_result
 
     def compute_result_annotation(self, *args_s):
@@ -1237,6 +1237,7 @@
     Assure the JIT that value is the same as const_value
     """
     assert value == const_value
+    return const_value
 
 def ll_record_exact_value(ll_value, ll_const_value):
     from rpython.rlib.debug import ll_assert
@@ -1244,27 +1245,33 @@
     from rpython.rtyper.lltypesystem import lltype
     ll_assert(ll_value == ll_const_value, "record_exact_value called with two different arguments")
     llop.jit_record_exact_value(lltype.Void, ll_value, ll_const_value)
+    return ll_const_value
 
 class Entry(ExtRegistryEntry):
     _about_ = record_exact_value
 
     def compute_result_annotation(self, s_val, s_const_val):
         from rpython.annotator import model as annmodel
-        annmodel.unionof(s_val, s_const_val) # produce error if types are incompatible
+        # produce error if types are incompatible
+        s_common = annmodel.unionof(s_val, s_const_val)
+        # we need to keep this annotation around for specialize_call().
+        # The easiest is to use this union as the return value.
+        return s_common
 
     def specialize_call(self, hop):
         from rpython.rtyper.lltypesystem import lltype
         from rpython.rtyper import rclass
 
-        v_inst = hop.inputarg(hop.args_r[0], arg=0)
-        v_const_inst = hop.inputarg(hop.args_r[1], arg=1)
+        r_common = hop.r_result    # from the union of the two annotations
+        v_inst = hop.inputarg(r_common, arg=0)
+        v_const_inst = hop.inputarg(r_common, arg=1)
         hop.exception_is_here()
         return hop.gendirectcall(ll_record_exact_value, v_inst, v_const_inst)
 
+
 def _jit_conditional_call(condition, function, *args):
     pass           # special-cased below
 
-
 @specialize.call_location()
 def conditional_call(condition, function, *args):
     """Does the same as:
diff --git a/rpython/rlib/test/test_jit.py b/rpython/rlib/test/test_jit.py
--- a/rpython/rlib/test/test_jit.py
+++ b/rpython/rlib/test/test_jit.py
@@ -370,6 +370,17 @@
             record_exact_value(a, a) # assume not crash
         self.interpret(g, [])
 
+    def test_record_exact_value_2(self):
+        class A(object):
+            pass
+        def make(j):
+            if j < 5: return None
+            return A()
+        def g(j):
+            a = make(j)
+            record_exact_value(a, None)
+        self.interpret(g, [3])
+
     def test_record_exact_value_int(self):
         @dont_look_inside
         def f():

From pypy.commits at gmail.com  Thu Oct 24 08:35:05 2019
From: pypy.commits at gmail.com (JulianB)
Date: Thu, 24 Oct 2019 05:35:05 -0700 (PDT)
Subject: [pypy-commit] pypy default: Update some links to HTTPS.
Message-ID: <5db19a79.1c69fb81.9c607.b5fa@mx.google.com>

Author: Julian Berman <Julian+Hg at GrayVines.com>
Branch: 
Changeset: r97845:6c7852e9615a
Date: 2019-10-24 08:32 -0400
http://bitbucket.org/pypy/pypy/changeset/6c7852e9615a/

Log:	Update some links to HTTPS.

diff --git a/README.rst b/README.rst
--- a/README.rst
+++ b/README.rst
@@ -9,15 +9,15 @@
 
 The home page for the interpreter is:
 
-    http://pypy.org/
+    https://pypy.org/
 
 If you want to help developing PyPy, this documentation might help you:
 
-    http://doc.pypy.org/
+    https://doc.pypy.org/
 
 More documentation about the RPython framework can be found here:
 
-    http://rpython.readthedocs.io/
+    https://rpython.readthedocs.io/
 
 The source for the documentation is in the pypy/doc directory.
 
@@ -25,7 +25,7 @@
 Using PyPy instead of CPython
 -----------------------------
 
-Please read the information at http://pypy.org/ to find the correct way to
+Please read the information at https://pypy.org/ to find the correct way to
 download and use PyPy as an alternative to CPython. 
 
 
@@ -36,7 +36,7 @@
 interpreter. It is time-consuming and requires significant computing resources.
 More information can be found here:
 
-    http://doc.pypy.org/en/latest/build.html
+    https://doc.pypy.org/en/latest/build.html
 
 Enjoy and send us feedback!
 

From pypy.commits at gmail.com  Thu Oct 24 08:35:07 2019
From: pypy.commits at gmail.com (JulianB)
Date: Thu, 24 Oct 2019 05:35:07 -0700 (PDT)
Subject: [pypy-commit] pypy default: Merge some license gap fixes.
Message-ID: <5db19a7b.1c69fb81.1eae1.d21b@mx.google.com>

Author: Julian Berman <Julian+Hg at GrayVines.com>
Branch: 
Changeset: r97846:f8f6fe914790
Date: 2019-10-24 08:33 -0400
http://bitbucket.org/pypy/pypy/changeset/f8f6fe914790/

Log:	Merge some license gap fixes.

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -3,10 +3,11 @@
 License
 =======
 
-Except when otherwise stated (look for LICENSE files in directories or
-information at the beginning of each file) all software and documentation in
-the 'rpython', 'pypy', 'ctype_configure', 'dotviewer', 'demo', 'lib_pypy',
-'py', and '_pytest' directories is licensed as follows:
+Except when otherwise stated (look for LICENSE files in directories
+or information at the beginning of each file) all software and
+documentation in the 'rpython', 'pypy', 'ctype_configure', 'dotviewer',
+'demo', 'extra_tests', 'include', 'lib_pypy', 'py', and '_pytest'
+directories is licensed as follows:
 
     The MIT License
 

From pypy.commits at gmail.com  Thu Oct 24 08:35:55 2019
From: pypy.commits at gmail.com (JulianB)
Date: Thu, 24 Oct 2019 05:35:55 -0700 (PDT)
Subject: [pypy-commit] pypy license-update: closing
Message-ID: <5db19aab.1c69fb81.be562.1d5d@mx.google.com>

Author: Julian Berman <Julian+Hg at GrayVines.com>
Branch: license-update
Changeset: r97847:068a29d3a9cd
Date: 2019-10-24 08:35 -0400
http://bitbucket.org/pypy/pypy/changeset/068a29d3a9cd/

Log:	closing


From pypy.commits at gmail.com  Thu Oct 24 08:37:04 2019
From: pypy.commits at gmail.com (JulianB)
Date: Thu, 24 Oct 2019 05:37:04 -0700 (PDT)
Subject: [pypy-commit] pypy https-readme: closing
Message-ID: <5db19af0.1c69fb81.3b871.61a0@mx.google.com>

Author: Julian Berman <Julian+Hg at GrayVines.com>
Branch: https-readme
Changeset: r97848:09f11925c361
Date: 2019-10-24 08:36 -0400
http://bitbucket.org/pypy/pypy/changeset/09f11925c361/

Log:	closing


From pypy.commits at gmail.com  Thu Oct 24 10:28:04 2019
From: pypy.commits at gmail.com (mattip)
Date: Thu, 24 Oct 2019 07:28:04 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge license-update into default
Message-ID: <5db1b4f4.1c69fb81.35717.d0b7@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r97849:9ecf0c6f29e5
Date: 2019-10-24 17:18 +0300
http://bitbucket.org/pypy/pypy/changeset/9ecf0c6f29e5/

Log:	merge license-update into default


From pypy.commits at gmail.com  Thu Oct 24 10:28:07 2019
From: pypy.commits at gmail.com (mattip)
Date: Thu, 24 Oct 2019 07:28:07 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge https-readme into default
Message-ID: <5db1b4f7.1c69fb81.b4f76.f2b0@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r97850:c32894ffed89
Date: 2019-10-24 17:19 +0300
http://bitbucket.org/pypy/pypy/changeset/c32894ffed89/

Log:	merge https-readme into default


From pypy.commits at gmail.com  Thu Oct 24 10:28:09 2019
From: pypy.commits at gmail.com (mattip)
Date: Thu, 24 Oct 2019 07:28:09 -0700 (PDT)
Subject: [pypy-commit] pypy default: update whatsnew-head.rst for new
 branches
Message-ID: <5db1b4f9.1c69fb81.5d686.df64@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r97851:01ba868b37c1
Date: 2019-10-24 17:26 +0300
http://bitbucket.org/pypy/pypy/changeset/01ba868b37c1/

Log:	update whatsnew-head.rst for new branches

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -9,3 +9,10 @@
 
 Fix segfault when calling descr-methods with no arguments
 
+.. branch: https-readme
+
+Convert http -> https in README.rst
+
+.. branch: license-update
+
+Update list directories in LICENSE

From pypy.commits at gmail.com  Thu Oct 24 15:23:14 2019
From: pypy.commits at gmail.com (stevie_92)
Date: Thu, 24 Oct 2019 12:23:14 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-gc-cycle: Improved stats
Message-ID: <5db1fa22.1c69fb81.899a7.2f17@mx.google.com>

Author: Stefan Beyer <home at sbeyer.at>
Branch: cpyext-gc-cycle
Changeset: r97852:4f393184be0e
Date: 2019-10-23 19:53 +0200
http://bitbucket.org/pypy/pypy/changeset/4f393184be0e/

Log:	Improved stats

diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -161,7 +161,7 @@
                     use_bytecode_counter=True)
             else:
                 module = space.builtin_modules['gc']
-                attribute = space.newtext('cpyext_durations')
+                attribute = space.newtext('cpyext_stat')
                 space.setattr(module, attribute, space.newlist([]))
 
                 pyobj_dealloc_action = PyObjDeallocAction(space)
@@ -321,26 +321,33 @@
                                              finalize, from_ref, cts)
     from pypy.module.cpyext.api import generic_cpy_call
 
-
+    #debug_print("rrc perform")
     if rawrefcount.check_state():
         start = time.time()
+        dead_calls = 0
+        cyclic_isolate_calls = 0
+        cyclic_garbage_calls = 0
+        garbage_calls = 0
 
         while True:
             py_obj = rawrefcount.next_dead(PyObject)
             if not py_obj:
                 break
+            dead_calls += 1
             decref(space, py_obj)
 
         while True:
             py_obj = rawrefcount.next_cyclic_isolate(PyObject)
             if not py_obj:
                 break
+            cyclic_isolate_calls += 1
             finalize(space, py_obj)
 
         while True:
             py_obj = rawrefcount.cyclic_garbage_head(PyObject)
             if not py_obj:
                 break
+            cyclic_garbage_calls += 1
             pyobj = rffi.cast(PyObject, py_obj)
             adr_int = llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(pyobj))
             pto = pyobj.c_ob_type
@@ -357,27 +364,38 @@
             if adr_int == llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(head)):
                 rawrefcount.cyclic_garbage_remove()
 
-        rawrefcount.begin_garbage()
-        w_list = space.newlist([])
-        while True:
-            w_obj = rawrefcount.next_garbage_pypy(W_Root)
-            if not py_obj:
-                break
-            w_list.append(w_obj)
-        while True:
-            w_pyobj = rawrefcount.next_garbage_pyobj(PyObject)
-            if not w_pyobj:
-                break
-            w_obj = from_ref(space, w_pyobj)
-            w_list.append(w_obj)
-        space.setattr(space.builtin_modules['gc'], space.newtext('garbage'),
-                      w_list)
-        rawrefcount.end_garbage()
+        # TODO: instead of building up this list every time, only do it in case something changed
+        if False: # TODO add check, if something changed
+            rawrefcount.begin_garbage()
+            w_list = space.newlist([]) # append to existing list???
+            while True:
+                w_obj = rawrefcount.next_garbage_pypy(W_Root)
+                if not w_obj:
+                    break
+                garbage_calls += 1
+                debug_print("append garbage pypy", w_obj)
+                w_list.append(w_obj)
+            while True:
+                w_pyobj = rawrefcount.next_garbage_pyobj(PyObject)
+                if not w_pyobj:
+                    break
+                garbage_calls += 1
+                w_obj = from_ref(space, w_pyobj)
+                debug_print("append garbage pyobj", w_obj)
+                w_list.append(w_obj)
+            space.setattr(space.builtin_modules['gc'],
+                          space.newtext('garbage'), w_list)
+            rawrefcount.end_garbage()
 
         duration = time.time() - start
+        list = space.newlist([space.newfloat(duration),
+                              space.newint(dead_calls),
+                              space.newint(cyclic_isolate_calls),
+                              space.newint(cyclic_garbage_calls),
+                              space.newint(garbage_calls)])
         module = space.builtin_modules['gc']
-        durations = space.getattr(module, space.newtext('cpyext_durations'))
-        durations.append(space.newfloat(duration))
+        durations = space.getattr(module, space.newtext('cpyext_stat'))
+        durations.append(list)
 
 class PyObjDeallocAction(executioncontext.AsyncAction):
     """An action that invokes _Py_Dealloc() on the dying PyObjects.
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -2026,7 +2026,7 @@
             #
             # Check that the flags are correct: we must not have
             # GCFLAG_TRACK_YOUNG_PTRS so far.
-            # TODO: fix
+            # TODO: fix (with rrc incmark, seems to work with rrc mark)
             #ll_assert(self.header(obj).tid & GCFLAG_TRACK_YOUNG_PTRS == 0,
             #          "old_objects_pointing_to_young contains obj with "
             #          "GCFLAG_TRACK_YOUNG_PTRS")
@@ -2411,7 +2411,13 @@
                 if self.rrc_enabled:
                     debug_print("starting rrc state:", self.rrc_gc.state)
                     debug_print("starting marking_state:", self.rrc_gc.marking_state)
+                    start_rrc = time.time()
+
                     rrc_finished = self.rrc_gc.major_collection_trace_step()
+
+                    duration_rrc = time.time() - start_rrc
+                    debug_print("rrc duration:", duration_rrc)
+
                     debug_print("ending rrc state:", self.rrc_gc.state)
                     debug_print("ending marking_state:", self.rrc_gc.marking_state)
                 else:
@@ -3146,6 +3152,7 @@
 
     def rawrefcount_check_state(self):
         ll_assert(self.rrc_enabled, "rawrefcount.init not called")
+        #debug_print("check state", self.rrc_gc.state)
         return self.rrc_gc.state == RawRefCountBaseGC.STATE_DEFAULT
 
     def rawrefcount_next_dead(self):
diff --git a/rpython/memory/gc/rrc/base.py b/rpython/memory/gc/rrc/base.py
--- a/rpython/memory/gc/rrc/base.py
+++ b/rpython/memory/gc/rrc/base.py
@@ -120,12 +120,12 @@
         self.tuple_maybe_untrack = tuple_maybe_untrack
         self.state = self.STATE_DEFAULT
         self.marking_state = 0
-        self.cycle_enabled = True
         inc_limit = env.read_uint_from_env('PYPY_RRC_GC_INCREMENT_STEP')
         if inc_limit > 0:
             self.inc_limit = inc_limit
         else:
             self.inc_limit = 1000
+        self.cycle_enabled = True
 
     def create_link_pypy(self, gcobj, pyobject):
         obj = llmemory.cast_ptr_to_adr(gcobj)
@@ -227,9 +227,9 @@
             # the first (c_gc_next) and last (c_gc_prev) pyobject in the list
             # of live objects that are garbage, so just fix the references
             list = self.pyobj_garbage_list
-            gchdr = list.c_gc_next
-            if list.c_gc_prev == gchdr:
-                list.c_gc_next = list # reached end of list, reset it
+            gchdr = list.c_gc_next # get next object in list
+            if list.c_gc_prev == gchdr: # if at end of list
+                list.c_gc_next = list # reset the list (now it is empty)
             else:
                 list.c_gc_next = gchdr.c_gc_next # move pointer foward
             return llmemory.cast_ptr_to_adr(self.gc_as_pyobj(gchdr))

From pypy.commits at gmail.com  Thu Oct 24 15:23:16 2019
From: pypy.commits at gmail.com (stevie_92)
Date: Thu, 24 Oct 2019 12:23:16 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-gc-cycle: Fixed performance issues in
 incmark
Message-ID: <5db1fa24.1c69fb81.ffe46.2ae5@mx.google.com>

Author: Stefan Beyer <home at sbeyer.at>
Branch: cpyext-gc-cycle
Changeset: r97853:4dc7f472f3df
Date: 2019-10-24 21:22 +0200
http://bitbucket.org/pypy/pypy/changeset/4dc7f472f3df/

Log:	Fixed performance issues in incmark Added debugging code

diff --git a/rpython/memory/gc/rrc/base.py b/rpython/memory/gc/rrc/base.py
--- a/rpython/memory/gc/rrc/base.py
+++ b/rpython/memory/gc/rrc/base.py
@@ -97,6 +97,8 @@
          self.GCFLAG_NO_HEAP_PTRS, self.GCFLAG_GARBAGE) = gc_flags
         self.p_list_young = self.gc.AddressStack()
         self.p_list_old = self.gc.AddressStack()
+        self.p_list_old_added = self.gc.AddressStack()
+        self.p_dict_old_free = self.gc.AddressDict()
         self.o_list_young = self.gc.AddressStack()
         self.o_list_old = self.gc.AddressStack()
         self.p_dict = self.gc.AddressDict()  # non-nursery keys only
@@ -139,6 +141,9 @@
             dct = self.p_dict
             if not self.gc.is_young_object(obj):
                 lst = self.p_list_old
+                if self.state == self.STATE_MARKING:
+                    debug_print("added p_list", pyobject)
+                    self.p_list_old_added.append(pyobject)
         lst.append(pyobject)
         dct.setitem(obj, pyobject)
 
@@ -259,14 +264,15 @@
         ll_assert(self.p_dict_nurs.length() == 0,
                   "p_dict_nurs not empty 1")
         lst = self.p_list_young
+        working_set = self.state == self.STATE_MARKING
         while lst.non_empty():
-            self._minor_free(lst.pop(), self.p_list_old, self.p_dict)
+            self._minor_free(lst.pop(), self.p_list_old, self.p_dict, working_set)
         lst = self.o_list_young
         no_o_dict = self.gc.null_address_dict()
         while lst.non_empty():
-            self._minor_free(lst.pop(), self.o_list_old, no_o_dict)
+            self._minor_free(lst.pop(), self.o_list_old, no_o_dict, False)
 
-    def _minor_free(self, pyobject, surviving_list, surviving_dict):
+    def _minor_free(self, pyobject, surviving_list, surviving_dict, working_set):
         intobj = self._pyobj(pyobject).c_ob_pypy_link
         obj = llmemory.cast_int_to_adr(intobj)
         if self.gc.is_in_nursery(obj):
@@ -299,6 +305,9 @@
         #
         if surviving:
             surviving_list.append(pyobject)
+            if working_set:
+                debug_print("added p_list", pyobject)
+                self.p_list_old_added.append(pyobject)
         else:
             self._free(pyobject)
 
@@ -309,6 +318,7 @@
         rc = self._pyobj(pyobject).c_ob_refcnt
         if rc >= REFCNT_FROM_PYPY_LIGHT:
             rc -= REFCNT_FROM_PYPY_LIGHT
+            debug_print("major_free", "free", pyobject, "refcnt", rc, "light")
             if rc == 0:
                 pygchdr = self.pyobj_as_gc(self._pyobj(pyobject))
                 if pygchdr <> lltype.nullptr(self.PYOBJ_GC_HDR):
@@ -325,6 +335,7 @@
             ll_assert(rc < int(REFCNT_FROM_PYPY_LIGHT * 0.99),
                       "refcount underflow from REFCNT_FROM_PYPY_LIGHT?")
             rc -= REFCNT_FROM_PYPY
+            debug_print("major_free", "free", pyobject, "refcnt", rc, "nonlight")
             self._pyobj(pyobject).c_ob_pypy_link = 0
             if rc == 0:
                 self.dealloc_pending.append(pyobject)
@@ -376,13 +387,15 @@
             pass  # the corresponding object may die
         else:
             # force the corresponding object to be alive
-            debug_print("pyobj stays alive", pyobj, "rc", rc, "cyclic_rc",
-                        cyclic_rc)
             if use_dict:
                 obj = self.pypy_link_dict.get(pyobject)
             else:
                 intobj = pyobj.c_ob_pypy_link
+                if intobj == 0:
+                    return
                 obj = llmemory.cast_int_to_adr(intobj)
+            debug_print("pyobj stays alive", pyobj, "rc", rc, "cyclic_rc",
+                        cyclic_rc)
             self.gc.objects_to_trace.append(obj)
             self.gc.visit_all_objects() # TODO: execute incrementally?
 
@@ -444,12 +457,14 @@
         #  * GCFLAG_NO_HEAP_PTRS: immortal object never traced (so far)
         intobj = self._pyobj(pyobject).c_ob_pypy_link
         obj = llmemory.cast_int_to_adr(intobj)
-        if self.gc.header(obj).tid & \
+        if intobj != 0 and self.gc.header(obj).tid & \
                 (self.GCFLAG_VISITED | self.GCFLAG_NO_HEAP_PTRS):
+            debug_print("major_free", "survive", pyobject)
             surviving_list.append(pyobject)
             if surviving_dict:
                 surviving_dict.insertclean(obj, pyobject)
         else:
+            debug_print("major_free", "free", pyobject)
             self._free(pyobject, True)
 
     def _untrack_tuples(self):
diff --git a/rpython/memory/gc/rrc/incmark.py b/rpython/memory/gc/rrc/incmark.py
--- a/rpython/memory/gc/rrc/incmark.py
+++ b/rpython/memory/gc/rrc/incmark.py
@@ -2,6 +2,7 @@
 from rpython.rtyper.lltypesystem import rffi
 from rpython.memory.gc.rrc.base import RawRefCountBaseGC
 from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop
+import time
 
 class RawRefCountIncMarkGC(RawRefCountBaseGC):
 
@@ -60,13 +61,21 @@
         # we are finished with marking, now finish things up
         ll_assert(self.state == self.STATE_GARBAGE_MARKING, "invalid state")
 
+        start = time.time()
+
         # sync snapshot
         self._sync_snapshot()
         self._debug_check_consistency(print_label="before-snap-discard")
 
+        debug_print("time snapshot sync", time.time() - start)
+        start = time.time()
+
         # now the snapshot is not needed any more, discard it
         self._discard_snapshot()
 
+        debug_print("time discard snapshot", time.time() - start)
+        start = time.time()
+
         # handle legacy finalizers (assumption: not a lot of legacy finalizers,
         # so no need to do it incrementally)
         if self._find_garbage(False):
@@ -74,17 +83,25 @@
             self._debug_check_consistency(print_label="end-legacy-fin")
         self.state = self.STATE_DEFAULT
 
+        debug_print("time legacy finalizer", time.time() - start)
+        start = time.time()
+
         # handle modern finalizers
         found_finalizer = self._find_finalizer()
         if found_finalizer:
             self._gc_list_move(self.pyobj_old_list, self.pyobj_isolate_list)
         use_cylicrc = not found_finalizer
 
+        debug_print("time modern finalizer", time.time() - start)
+        start = time.time()
+
         # now mark all pypy objects at the border, depending on the results
         self._debug_check_consistency(print_label="end-mark-cyclic")
         debug_print("use_cylicrc", use_cylicrc)
         self.p_list_old.foreach(self._major_trace, (use_cylicrc, False))
         self._debug_check_consistency(print_label="end-mark")
+
+        debug_print("time mark p_list_old", time.time() - start)
         return True
 
     def _sync_snapshot(self):
@@ -98,11 +115,12 @@
         consistent = True
         self.snapshot_consistent = True
 
+        start = time.time()
+
         # sync p_list_old (except gc-objects)
         # simply iterate the snapshot for objects in p_list, as linked objects
         # might not be freed, except by the gc; p_list is always at the
         # beginning of the snapshot, so break if we reached a different pyobj
-        free_p_list = self.gc.AddressStack()
         for i in range(0, self.total_objs):
             snapobj = self.snapshot_objs[i]
             if snapobj.pypy_link == 0:
@@ -123,8 +141,17 @@
                 if not consistent:
                     break
                 # move to separate list
-                self.p_list_old.remove(snapobj.pyobj)
-                free_p_list.append(snapobj.pyobj)
+                #self.p_list_old.remove(snapobj.pyobj) # TODO: this might be evil... do something different... -> unlink? special link?
+
+                # remove link, to free non-gc (so they won't get marked and are freed)
+                pyobj = llmemory.cast_adr_to_ptr(snapobj.pyobj, self.PYOBJ_HDR_PTR)
+                link = llmemory.cast_int_to_adr(pyobj.c_ob_pypy_link)
+                self.p_dict_old_free.setitem(snapobj.pyobj, link)
+                pyobj.c_ob_pypy_link = 0
+                debug_print("free p_list", pyobj, "refcnt", pyobj.c_ob_refcnt)
+
+        debug_print("time sync p_list_old", time.time() - start)
+        start = time.time()
 
         # look if there is a (newly) linked non-gc proxy, where the non-rc obj
         # is unmarked
@@ -132,6 +159,9 @@
         self.p_list_old.foreach(self._check_consistency_p_list_old, None)
         consistent &= self.p_list_old_consistent
 
+        debug_print("time consistency p_list_old", time.time() - start)
+        start = time.time()
+
         # sync gc objects
         pygchdr = self.pyobj_list.c_gc_next
         while pygchdr <> self.pyobj_list and consistent:
@@ -152,11 +182,18 @@
                     if not (self.gc.header(addr).tid &
                             (self.GCFLAG_VISITED | self.GCFLAG_NO_HEAP_PTRS)):
                         consistent = False
+                        debug_print("inconsistency found",
+                                    "marked new object", pyobj)
                         break
             pygchdr = next_old
         pygchdr_continue_gc = pygchdr
 
+
+        debug_print("time sync gc objs", time.time() - start)
+        start = time.time()
+
         # sync isolate objs
+        debug_print("check isolate")
         isolate_consistent = True
         pygchdr = self.pyobj_isolate_old_list.c_gc_next
         while pygchdr <> self.pyobj_isolate_old_list and isolate_consistent:
@@ -169,12 +206,15 @@
         consistent &= isolate_consistent
         self._debug_check_consistency(print_label="end-check-consistency")
 
+        debug_print("time sync isolate objs", time.time() - start)
+        start = time.time()
+
         if consistent:
-            free_p_list.foreach(self._free_p_list, None)
+            debug_print("consistent")
         else:
-            # keep linked non-gc alive
-            while free_p_list.non_empty():
-                self.p_list_old.append(free_p_list.pop())
+            debug_print("inconsistent")
+            # fix link
+            self.p_dict_old_free.foreach(self._fix_p_list, None)
             # continue previous loop, keep objects alive
             pygchdr = pygchdr_continue_gc
             while pygchdr <> self.pyobj_list:
@@ -191,6 +231,12 @@
             if not self._gc_list_is_empty(self.pyobj_old_list):
                 self._gc_list_merge(self.pyobj_old_list, self.pyobj_list)
 
+        self.p_dict_old_free.delete()
+        self.p_dict_old_free = self.gc.AddressDict()
+
+        debug_print("time free/inconsistent", time.time() - start)
+        start = time.time()
+
         if isolate_consistent:
             if not self._gc_list_is_empty(self.pyobj_isolate_old_list):
                 self._gc_list_merge(self.pyobj_isolate_old_list,
@@ -218,6 +264,8 @@
                 self._gc_list_merge(self.pyobj_isolate_dead_list,
                                     self.pyobj_list)
 
+        debug_print("time sync isolate", time.time() - start)
+
     def _check_consistency_gc(self, pygchdr, pylist_dead_target):
         c_gc_refs = self._pyobj_gc_refcnt_get(pygchdr)
         snapobj = self.snapshot_objs[c_gc_refs - 1]
@@ -230,6 +278,7 @@
             # refcount equal
             consistent = snapobj.refcnt_original == pyobj.c_ob_refcnt
             if not consistent:
+                debug_print("inconsistency found", "refcount not equal", pyobj)
                 return False
             # outgoing (internal) references equal
             self.snapshot_curr = snapobj
@@ -237,6 +286,7 @@
             self._check_snapshot_traverse(pyobj)
             consistent = self.snapshot_consistent
             if not consistent:
+                debug_print("inconsistency found", "references not equal", pyobj)
                 return False
             # consistent -> prepare object for collection
             self._gc_list_remove(pygchdr)
@@ -262,18 +312,11 @@
             if not (self.gc.header(addr).tid &
                     (self.GCFLAG_VISITED | self.GCFLAG_NO_HEAP_PTRS)):
                 self.p_list_old_consistent = False
+                debug_print("inconsistency found", "marked p_list", pyobj)
 
-    def _free_p_list(self, pyobject, foo):
-        from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY
-        from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY_LIGHT
+    def _fix_p_list(self, pyobject, addr_link, foo):
         pyobj = llmemory.cast_adr_to_ptr(pyobject, self.PYOBJ_HDR_PTR)
-        refcnt = pyobj.c_ob_refcnt
-        if refcnt >= REFCNT_FROM_PYPY_LIGHT:
-            refcnt -= REFCNT_FROM_PYPY_LIGHT
-        elif refcnt >= REFCNT_FROM_PYPY:
-            refcnt -= REFCNT_FROM_PYPY
-        pyobj.c_ob_refcnt = refcnt
-        pyobj.c_ob_pypy_link = 0
+        pyobj.c_ob_pypy_link = llmemory.cast_adr_to_int(addr_link, "symbolic")
 
     def _collect_roots(self):
         # Subtract all internal refcounts from the cyclic refcount
@@ -298,6 +341,13 @@
         simple_limit = 0
         first = True # rescan proxies, in case only non-rc have been marked
         #
+        while self.p_list_old_added.non_empty():
+            addr = self.p_list_old_added.pop()
+            pyobj = llmemory.cast_adr_to_ptr(addr, self.PYOBJ_HDR_PTR)
+            obj = llmemory.cast_int_to_adr(pyobj.c_ob_pypy_link)
+            debug_print("trace p_list", pyobj)
+            self.gc.objects_to_trace.append(obj)
+        #
         while first or (self.pyobj_to_trace.non_empty() and not reached_limit):
             while self.pyobj_to_trace.non_empty() and not reached_limit:
                 addr = self.pyobj_to_trace.pop()
@@ -308,7 +358,7 @@
                 simple_limit += 1
                 if simple_limit > self.inc_limit: # TODO: add test
                     reached_limit = True
-
+            self.gc.visit_all_objects()  # TODO: implement sane limit (ex. half of normal limit), retrace proxies
             self.p_list_old.foreach(self._mark_rawrefcount_linked, None)
             self.o_list_old.foreach(self._mark_rawrefcount_linked, None)
             first = False
@@ -343,7 +393,6 @@
                 intobj = snapobj.pypy_link
                 obj = llmemory.cast_int_to_adr(intobj)
                 self.gc.objects_to_trace.append(obj)
-                self.gc.visit_all_objects()  # TODO: move to outer loop, implement sane limit (ex. half of normal limit), retrace proxies
             # mark as processed
             snapobj.status = 0
         return alive
@@ -553,6 +602,7 @@
         pygchdr = self.pyobj_as_gc(pyobj)
         # only for non-gc
         if pygchdr == lltype.nullptr(self.PYOBJ_GC_HDR):
+            debug_print("clear link", pyobj)
             link = llmemory.cast_int_to_adr(pyobj.c_ob_pypy_link)
             self.pypy_link_dict.setitem(pyobject, link)
             pyobj.c_ob_pypy_link = 0

From pypy.commits at gmail.com  Fri Oct 25 03:05:34 2019
From: pypy.commits at gmail.com (mattip)
Date: Fri, 25 Oct 2019 00:05:34 -0700 (PDT)
Subject: [pypy-commit] pypy default: update bundled pip,
 setuptools so we can use manylinux2010 wheels
Message-ID: <5db29ebe.1c69fb81.1b312.5631@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r97854:dca2f786de74
Date: 2019-10-25 07:16 +0300
http://bitbucket.org/pypy/pypy/changeset/dca2f786de74/

Log:	update bundled pip, setuptools so we can use manylinux2010 wheels

diff --git a/lib-python/2.7/ensurepip/__init__.py b/lib-python/2.7/ensurepip/__init__.py
--- a/lib-python/2.7/ensurepip/__init__.py
+++ b/lib-python/2.7/ensurepip/__init__.py
@@ -12,9 +12,9 @@
 __all__ = ["version", "bootstrap"]
 
 
-_SETUPTOOLS_VERSION = "28.8.0"
+_SETUPTOOLS_VERSION = "41.2.0"
 
-_PIP_VERSION = "9.0.1"
+_PIP_VERSION = "19.2.3"
 
 _PROJECTS = [
     ("setuptools", _SETUPTOOLS_VERSION),
diff --git a/lib-python/2.7/ensurepip/_bundled/pip-19.2.3-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-19.2.3-py2.py3-none-any.whl
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..8118df8ac1940f8c6cb410fbc18e5fae59872b95
GIT binary patch

[cut]
diff --git a/lib-python/2.7/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl
deleted file mode 100644
index 4b8ecc69db7e37fc6dd7b6dd8f690508f42866a1..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch

[cut]

diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl
deleted file mode 100644
index 502e3cb418c154872ad6e677ef8b63557b38ec35..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch

[cut]

diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-41.2.0-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-41.2.0-py2.py3-none-any.whl
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..82df6f63f4ee97380af0a29d8825ae775333b86d
GIT binary patch

[cut]


From pypy.commits at gmail.com  Fri Oct 25 03:05:36 2019
From: pypy.commits at gmail.com (mattip)
Date: Fri, 25 Oct 2019 00:05:36 -0700 (PDT)
Subject: [pypy-commit] pypy default: change the DEFAULT_SOABI when the C-API
 changes (now linked to the PYPY_VERSION)
Message-ID: <5db29ec0.1c69fb81.7df97.6926@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r97855:b8d283f0ad3e
Date: 2019-10-25 08:25 +0300
http://bitbucket.org/pypy/pypy/changeset/b8d283f0ad3e/

Log:	change the DEFAULT_SOABI when the C-API changes (now linked to the
	PYPY_VERSION)

diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -32,14 +32,10 @@
 
 SO = '.pyd' if _WIN32 else '.so'
 
-# this used to change for every minor version, but no longer does: there
-# is little point any more, as the so's tend to be cross-version-
-# compatible, more so than between various versions of CPython.  Be
-# careful if we need to update it again: it is now used for both cpyext
-# and cffi so's.  If we do have to update it, we'd likely need a way to
+# Be careful update when changing this: it is now used for both cpyext
+# and cffi so's. If we do have to update it, we'd likely need a way to
 # split the two usages again.
-#DEFAULT_SOABI = 'pypy-%d%d' % PYPY_VERSION[:2]
-DEFAULT_SOABI = 'pypy-41'
+DEFAULT_SOABI = 'pypy-%d%d' % PYPY_VERSION[:2]
 
 @specialize.memo()
 def get_so_extension(space):

From pypy.commits at gmail.com  Fri Oct 25 03:05:38 2019
From: pypy.commits at gmail.com (mattip)
Date: Fri, 25 Oct 2019 00:05:38 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: update bundled pip,
 setuptools so we can use manylinux2010 wheels
Message-ID: <5db29ec2.1c69fb81.78281.656f@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r97856:694f1a91de88
Date: 2019-10-25 08:30 +0300
http://bitbucket.org/pypy/pypy/changeset/694f1a91de88/

Log:	update bundled pip, setuptools so we can use manylinux2010 wheels

diff --git a/lib-python/3/ensurepip/__init__.py b/lib-python/3/ensurepip/__init__.py
--- a/lib-python/3/ensurepip/__init__.py
+++ b/lib-python/3/ensurepip/__init__.py
@@ -8,9 +8,9 @@
 __all__ = ["version", "bootstrap"]
 
 
-_SETUPTOOLS_VERSION = "40.6.2"
+_SETUPTOOLS_VERSION = "41.2.0"
 
-_PIP_VERSION = "18.1"
+_PIP_VERSION = "19.2.3"
 
 _PROJECTS = [
     ("setuptools", _SETUPTOOLS_VERSION),
diff --git a/lib-python/3/ensurepip/_bundled/pip-18.1-py2.py3-none-any.whl b/lib-python/3/ensurepip/_bundled/pip-18.1-py2.py3-none-any.whl
deleted file mode 100644
index c3c146f6da272399a19fd6e21364a4f45066cec7..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch

[cut]

diff --git a/lib-python/3/ensurepip/_bundled/pip-19.2.3-py2.py3-none-any.whl b/lib-python/3/ensurepip/_bundled/pip-19.2.3-py2.py3-none-any.whl
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..8118df8ac1940f8c6cb410fbc18e5fae59872b95
GIT binary patch

[cut]
diff --git a/lib-python/3/ensurepip/_bundled/setuptools-40.6.2-py2.py3-none-any.whl b/lib-python/3/ensurepip/_bundled/setuptools-40.6.2-py2.py3-none-any.whl
deleted file mode 100644
index 4c8a619571d1e50ae8b6b961a4bebc32c90694e1..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch

[cut]

diff --git a/lib-python/3/ensurepip/_bundled/setuptools-41.2.0-py2.py3-none-any.whl b/lib-python/3/ensurepip/_bundled/setuptools-41.2.0-py2.py3-none-any.whl
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..82df6f63f4ee97380af0a29d8825ae775333b86d
GIT binary patch

[cut]


From pypy.commits at gmail.com  Fri Oct 25 03:05:40 2019
From: pypy.commits at gmail.com (mattip)
Date: Fri, 25 Oct 2019 00:05:40 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: the pyc filename is not dependent on the
 pypy version; do like cpython
Message-ID: <5db29ec4.1c69fb81.c2f3.57d3@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r97857:2dca56b6a653
Date: 2019-10-25 10:01 +0300
http://bitbucket.org/pypy/pypy/changeset/2dca56b6a653/

Log:	the pyc filename is not dependent on the pypy version; do like
	cpython

diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -16,7 +16,7 @@
 from rpython.rlib.objectmodel import we_are_translated, specialize
 from rpython.rlib.signature import signature
 from rpython.rlib import rposix_stat, types
-from pypy.module.sys.version import PYPY_VERSION
+from pypy.module.sys.version import PYPY_VERSION, CPYTHON_VERSION
 
 _WIN32 = sys.platform == 'win32'
 
@@ -24,7 +24,7 @@
 PREFIX = 'pypy3-'
 DEFAULT_SOABI_BASE = '%s%d%d' % ((PREFIX,) + PYPY_VERSION[:2])
 
-PYC_TAG = '%s%d%d' % ((PREFIX,) + PYPY_VERSION[:2])   # 'pypy3-XY'
+PYC_TAG = '%s%d%d' % ((PREFIX,) + CPYTHON_VERSION[:2])   # 'pypy3-XY'
 
 # see also pypy_incremental_magic in interpreter/pycode.py for the magic
 # version number stored inside pyc files.
diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py
--- a/pypy/module/imp/test/test_app.py
+++ b/pypy/module/imp/test/test_app.py
@@ -315,4 +315,4 @@
         import sys
         if not hasattr(sys, 'pypy_version_info'):
             skip('This test is PyPy-only')
-        assert imp.get_tag() == 'pypy3-%d%d' % sys.pypy_version_info[0:2]
+        assert imp.get_tag() == 'pypy3-%d%d' % sys.version_info[0:2]

From pypy.commits at gmail.com  Fri Oct 25 04:26:29 2019
From: pypy.commits at gmail.com (mattip)
Date: Fri, 25 Oct 2019 01:26:29 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: reduce difference to cpython 3.6.9
Message-ID: <5db2b1b5.1c69fb81.1796e.5dcd@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r97858:a1c8dde72521
Date: 2019-10-25 11:11 +0300
http://bitbucket.org/pypy/pypy/changeset/a1c8dde72521/

Log:	reduce difference to cpython 3.6.9

diff --git a/lib-python/3/distutils/command/build_ext.py b/lib-python/3/distutils/command/build_ext.py
--- a/lib-python/3/distutils/command/build_ext.py
+++ b/lib-python/3/distutils/command/build_ext.py
@@ -28,11 +28,6 @@
     from distutils.ccompiler import show_compilers
     show_compilers()
 
-def _get_c_extension_suffix():
-    import importlib
-    suffixes = importlib.machinery.EXTENSION_SUFFIXES
-    return suffixes[0] if suffixes else None
-
 
 class build_ext(Command):
 
@@ -686,15 +681,7 @@
         """
         from distutils.sysconfig import get_config_var
         ext_path = ext_name.split('.')
-        # PyPy tweak: first try to get the C extension suffix from
-        # 'imp'.  If it fails we fall back to the 'SO' config var, like
-        # the previous version of this code did.  This should work for
-        # CPython too.  The point is that on PyPy with cpyext, the
-        # config var 'SO' is just ".so" but we want to return
-        # ".pypy-VERSION.so" instead.
-        ext_suffix = _get_c_extension_suffix()
-        if ext_suffix is None:
-            ext_suffix = get_config_var('EXT_SUFFIX')     # fall-back
+        ext_suffix = get_config_var('EXT_SUFFIX')
         return os.path.join(*ext_path) + ext_suffix
 
     def get_export_symbols(self, ext):

From pypy.commits at gmail.com  Fri Oct 25 08:04:33 2019
From: pypy.commits at gmail.com (arigo)
Date: Fri, 25 Oct 2019 05:04:33 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue #3100
Message-ID: <5db2e4d1.1c69fb81.8a572.fd00@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r97859:ce2c72abde3a
Date: 2019-10-25 14:01 +0200
http://bitbucket.org/pypy/pypy/changeset/ce2c72abde3a/

Log:	Issue #3100

	Check in test in 'default', although it passes here, to be sure we
	don't get a regression later

diff --git a/pypy/objspace/std/test/test_newformat.py b/pypy/objspace/std/test/test_newformat.py
--- a/pypy/objspace/std/test/test_newformat.py
+++ b/pypy/objspace/std/test/test_newformat.py
@@ -199,6 +199,13 @@
         assert self.s('{0:\x00<12}').format(3+2.0j) == '(3+2j)' + '\x00' * 6
         assert self.s('{0:\x01<12}').format(3+2.0j) == '(3+2j)' + '\x01' * 6
 
+    def test_issue3100(self):
+        class Foo:
+            def __format__(self, f):
+                return '<<%r>>' % (f,)
+        fmtstr = self.s("{:[XYZ}")
+        assert fmtstr.format(Foo()) == "<<%r>>" % (self.s("[XYZ"),)
+
 
 class AppTestUnicodeFormat(BaseStringFormatTests):
     def setup_class(cls):

From pypy.commits at gmail.com  Fri Oct 25 08:04:35 2019
From: pypy.commits at gmail.com (arigo)
Date: Fri, 25 Oct 2019 05:04:35 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: Issue #3100
Message-ID: <5db2e4d3.1c69fb81.e94ec.906e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.6
Changeset: r97860:82e53786ebea
Date: 2019-10-25 14:03 +0200
http://bitbucket.org/pypy/pypy/changeset/82e53786ebea/

Log:	Issue #3100

	Test and fix

diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py
--- a/pypy/objspace/std/newformat.py
+++ b/pypy/objspace/std/newformat.py
@@ -117,6 +117,7 @@
                     nested = 1
                     field_start = i
                     recursive = False
+                    in_second_part = False
                     while i < end:
                         c = s[i]
                         if c == "{":
@@ -126,11 +127,13 @@
                             nested -= 1
                             if not nested:
                                 break
-                        elif c == "[":
+                        elif c == "[" and not in_second_part:
                             i += 1
                             while i < end and s[i] != "]":
                                 i += 1
                             continue
+                        elif c == ':' or c == '!':
+                            in_second_part = True
                         i += 1
                     if nested:
                         raise oefmt(space.w_ValueError, "Unmatched '{'")
diff --git a/pypy/objspace/std/test/test_newformat.py b/pypy/objspace/std/test/test_newformat.py
--- a/pypy/objspace/std/test/test_newformat.py
+++ b/pypy/objspace/std/test/test_newformat.py
@@ -203,6 +203,13 @@
         raises(ValueError, self.s("{a{b}").format, 42)
         raises(ValueError, self.s("{[}").format, 42)
 
+    def test_issue3100(self):
+        class Foo:
+            def __format__(self, f):
+                return '<<%r>>' % (f,)
+        fmtstr = self.s("{:[XYZ}")
+        assert fmtstr.format(Foo()) == "<<%r>>" % (self.s("[XYZ"),)
+
 
 class AppTestUnicodeFormat(BaseStringFormatTests):
     def setup_class(cls):

From pypy.commits at gmail.com  Fri Oct 25 08:35:01 2019
From: pypy.commits at gmail.com (arigo)
Date: Fri, 25 Oct 2019 05:35:01 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue #3094
Message-ID: <5db2ebf5.1c69fb81.2761a.0177@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r97861:da8096b43023
Date: 2019-10-25 14:34 +0200
http://bitbucket.org/pypy/pypy/changeset/da8096b43023/

Log:	Issue #3094

	Simple workaround for programs calling
	'sys.setrecursionlimit(huge_value)'

diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -59,6 +59,12 @@
     from rpython.rlib.rgc import increase_root_stack_depth
     if new_limit <= 0:
         raise oefmt(space.w_ValueError, "recursion limit must be positive")
+    # Some programs use very large values to mean "don't check, I want to
+    # use as much as possible and then segfault".  Add a silent upper bound
+    # of 10**6 here, because huge values cause huge shadowstacks to be
+    # allocated (or MemoryErrors).
+    if new_limit > 1000000:
+        new_limit = 1000000
     space.sys.recursionlimit = new_limit
     _stack_set_length_fraction(new_limit * 0.001)
     increase_root_stack_depth(int(new_limit * 0.001 * 163840))

From pypy.commits at gmail.com  Fri Oct 25 09:38:07 2019
From: pypy.commits at gmail.com (arigo)
Date: Fri, 25 Oct 2019 06:38:07 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: hg merge default
Message-ID: <5db2fabf.1c69fb81.a5bd8.c40e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.6
Changeset: r97862:34cc698bbcd1
Date: 2019-10-25 15:34 +0200
http://bitbucket.org/pypy/pypy/changeset/34cc698bbcd1/

Log:	hg merge default

	This includes the changes to
	unicodehelper._str_decode_utf8_slowpath(). If these changes were not
	meant to be merged, just revert that part.

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -3,10 +3,11 @@
 License
 =======
 
-Except when otherwise stated (look for LICENSE files in directories or
-information at the beginning of each file) all software and documentation in
-the 'rpython', 'pypy', 'ctype_configure', 'dotviewer', 'demo', 'lib_pypy',
-'py', and '_pytest' directories is licensed as follows:
+Except when otherwise stated (look for LICENSE files in directories
+or information at the beginning of each file) all software and
+documentation in the 'rpython', 'pypy', 'ctype_configure', 'dotviewer',
+'demo', 'extra_tests', 'include', 'lib_pypy', 'py', and '_pytest'
+directories is licensed as follows:
 
     The MIT License
 
diff --git a/README.rst b/README.rst
--- a/README.rst
+++ b/README.rst
@@ -9,15 +9,15 @@
 
 The home page for the interpreter is:
 
-    http://pypy.org/
+    https://pypy.org/
 
 If you want to help developing PyPy, this documentation might help you:
 
-    http://doc.pypy.org/
+    https://doc.pypy.org/
 
 More documentation about the RPython framework can be found here:
 
-    http://rpython.readthedocs.io/
+    https://rpython.readthedocs.io/
 
 The source for the documentation is in the pypy/doc directory.
 
@@ -25,7 +25,7 @@
 Using PyPy instead of CPython
 -----------------------------
 
-Please read the information at http://pypy.org/ to find the correct way to
+Please read the information at https://pypy.org/ to find the correct way to
 download and use PyPy as an alternative to CPython. 
 
 
@@ -36,7 +36,7 @@
 interpreter. It is time-consuming and requires significant computing resources.
 More information can be found here:
 
-    http://doc.pypy.org/en/latest/build.html
+    https://doc.pypy.org/en/latest/build.html
 
 Enjoy and send us feedback!
 
diff --git a/lib-python/2.7/ensurepip/_bundled/pip-19.2.3-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-19.2.3-py2.py3-none-any.whl
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..8118df8ac1940f8c6cb410fbc18e5fae59872b95
GIT binary patch

[cut]
diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-41.2.0-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-41.2.0-py2.py3-none-any.whl
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..82df6f63f4ee97380af0a29d8825ae775333b86d
GIT binary patch

[cut]

diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -2,8 +2,18 @@
 import time
 import _thread
 import weakref
-from _pypy_openssl import ffi
-from _pypy_openssl import lib
+
+try:
+    from _pypy_openssl import ffi
+    from _pypy_openssl import lib
+except ImportError as e:
+    import os
+    msg = "\n\nThe _ssl cffi module either doesn't exist or is incompatible with your machine's shared libraries.\n" + \
+          "If you have a compiler installed, you can try to rebuild it by running:\n" + \
+          "cd %s\n" % os.path.abspath(os.path.dirname(os.path.dirname(__file__))) + \
+          "%s _ssl_build.py\n" % sys.executable
+    raise ImportError(str(e) + msg)
+
 from _cffi_ssl._stdssl.certificate import (_test_decode_cert,
     _decode_certificate, _certificate_to_der)
 from _cffi_ssl._stdssl.utility import (_str_with_len, _bytes_with_len,
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -41,7 +41,7 @@
     "_multibytecodec", "_continuation", "_cffi_backend",
     "_csv", "_pypyjson", "_posixsubprocess", "_cppyy", # "micronumpy",
     "_jitlog",
-    #" _ssl", "_hashlib", "crypt"
+    # "_hashlib", "crypt"
 ])
 
 import rpython.rlib.rvmprof.cintf
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -9,3 +9,10 @@
 
 Fix segfault when calling descr-methods with no arguments
 
+.. branch: https-readme
+
+Convert http -> https in README.rst
+
+.. branch: license-update
+
+Update list directories in LICENSE
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -7,6 +7,7 @@
 from rpython.rlib.rarithmetic import r_uint, intmask
 from rpython.rtyper.lltypesystem import rffi
 from pypy.module.unicodedata.interp_ucd import unicodedb
+from rpython.rlib import runicode
 
 @specialize.memo()
 def decode_error_handler(space):
@@ -56,7 +57,6 @@
 
 def fsdecode(space, w_string):
     from pypy.module._codecs import interp_codecs
-    from rpython.rlib import runicode
     state = space.fromcache(interp_codecs.CodecState)
     errorhandler=state.decode_error_handler
     if _WIN32:
@@ -368,7 +368,6 @@
 
     def str_decode_mbcs(s, errors, final, errorhandler, force_ignore=True):
         slen = len(s)
-        from rpython.rlib import runicode
         res, size = runicode.str_decode_mbcs(s, slen, errors, final=final,
                            errorhandler=errorhandler, force_ignore=force_ignore)
         res_utf8 = runicode.unicode_encode_utf_8(res, size, 'strict')
@@ -389,139 +388,150 @@
     """
     if errors is None:
         errors = 'strict'
-    slen = len(s)
-    res = StringBuilder(slen)
+    size = len(s)
+    result = StringBuilder(size)
     pos = 0
-    end = len(s)
-    while pos < end:
+    while pos < size:
         ordch1 = ord(s[pos])
         # fast path for ASCII
+        # XXX maybe use a while loop here
         if ordch1 <= 0x7F:
             pos += 1
-            res.append(chr(ordch1))
+            result.append(chr(ordch1))
             continue
 
-        if ordch1 <= 0xC1:
-            r, pos, rettype = errorhandler(errors, "utf-8", "invalid start byte",
-                    s, pos, pos + 1)
-            res.append(r)
-            continue
+        n = ord(runicode._utf8_code_length[ordch1 - 0x80])
+        if pos + n > size:
+            if not final:
+                break
+            # argh, this obscure block of code is mostly a copy of
+            # what follows :-(
+            charsleft = size - pos - 1 # either 0, 1, 2
+            # note: when we get the 'unexpected end of data' we need
+            # to care about the pos returned; it can be lower than size,
+            # in case we need to continue running this loop
+            if not charsleft:
+                # there's only the start byte and nothing else
+                r, pos, rettype = errorhandler(errors, 'utf-8',
+                                      'unexpected end of data',
+                                      s, pos, pos+1)
+                result.append(r)
+                continue
+            ordch2 = ord(s[pos+1])
+            if n == 3:
+                # 3-bytes seq with only a continuation byte
+                if rutf8._invalid_byte_2_of_3(ordch1, ordch2, allow_surrogates):
+                    # second byte invalid, take the first and continue
+                    r, pos, rettype = errorhandler(errors, 'utf-8',
+                                          'invalid continuation byte',
+                                          s, pos, pos+1)
+                    result.append(r)
+                    continue
+                else:
+                    # second byte valid, but third byte missing
+                    r, pos, rettype = errorhandler(errors, 'utf-8',
+                                      'unexpected end of data',
+                                      s, pos, pos+2)
+                    result.append(r)
+                    continue
+            elif n == 4:
+                # 4-bytes seq with 1 or 2 continuation bytes
+                if rutf8._invalid_byte_2_of_4(ordch1, ordch2):
+                    # second byte invalid, take the first and continue
+                    r, pos, rettype = errorhandler(errors, 'utf-8',
+                                          'invalid continuation byte',
+                                          s, pos, pos+1)
+                    result.append(r)
+                    continue
+                elif charsleft == 2 and rutf8._invalid_byte_3_of_4(ord(s[pos+2])):
+                    # third byte invalid, take the first two and continue
+                    r, pos, rettype = errorhandler(errors, 'utf-8',
+                                          'invalid continuation byte',
+                                          s, pos, pos+2)
+                    result.append(r)
+                    continue
+                else:
+                    # there's only 1 or 2 valid cb, but the others are missing
+                    r, pos, rettype = errorhandler(errors, 'utf-8',
+                                      'unexpected end of data',
+                                      s, pos, pos+charsleft+1)
+                    result.append(r)
+                    continue
+            raise AssertionError("unreachable")
 
-        pos += 1
+        if n == 0:
+            r, pos, rettype = errorhandler(errors, 'utf-8',
+                                  'invalid start byte',
+                                  s, pos, pos+1)
+            result.append(r)
 
-        if ordch1 <= 0xDF:
-            if pos >= end:
-                if not final:
-                    pos -= 1
-                    break
-                r, pos, rettype = errorhandler(errors, "utf-8", "unexpected end of data",
-                    s, pos - 1, pos)
-                res.append(r)
-                continue
-            ordch2 = ord(s[pos])
+        elif n == 1:
+            assert 0, "ascii should have gone through the fast path"
 
+        elif n == 2:
+            ordch2 = ord(s[pos+1])
             if rutf8._invalid_byte_2_of_2(ordch2):
-                r, pos, rettype = errorhandler(errors, "utf-8", "invalid continuation byte",
-                    s, pos - 1, pos)
-                res.append(r)
+                r, pos, rettype = errorhandler(errors, 'utf-8',
+                                      'invalid continuation byte',
+                                      s, pos, pos+1)
+                result.append(r)
                 continue
             # 110yyyyy 10zzzzzz -> 00000000 00000yyy yyzzzzzz
-            pos += 1
-            res.append(chr(ordch1))
-            res.append(chr(ordch2))
-            continue
+            result.append(chr(ordch1))
+            result.append(chr(ordch2))
+            pos += 2
 
-        if ordch1 <= 0xEF:
-            if (pos + 2) > end:
-                if not final:
-                    pos -= 1
-                    break
-                if (pos) < end and  rutf8._invalid_byte_2_of_3(ordch1,
-                                                ord(s[pos]), allow_surrogates):
-                    msg = "invalid continuation byte"
-                    r, pos, rettype = errorhandler(errors, "utf-8", msg, s,
-                                                   pos - 1, pos)
-                else:
-                    msg = "unexpected end of data"
-                    r, pos, rettype = errorhandler(errors, "utf-8", msg, s,
-                                                   pos - 1, pos)
-                    pos = end
-                res.append(r)
-                continue
-            ordch2 = ord(s[pos])
-            ordch3 = ord(s[pos + 1])
-
+        elif n == 3:
+            ordch2 = ord(s[pos+1])
+            ordch3 = ord(s[pos+2])
             if rutf8._invalid_byte_2_of_3(ordch1, ordch2, allow_surrogates):
-                r, pos, rettype = errorhandler(errors, "utf-8", "invalid continuation byte",
-                    s, pos - 1, pos)
-                res.append(r)
+                r, pos, rettype = errorhandler(errors, 'utf-8',
+                                      'invalid continuation byte',
+                                      s, pos, pos+1)
+                result.append(r)
                 continue
             elif rutf8._invalid_byte_3_of_3(ordch3):
-                r, pos, rettype = errorhandler(errors, "utf-8", "invalid continuation byte",
-                    s, pos - 1, pos + 1)
-                res.append(r)
+                r, pos, rettype = errorhandler(errors, 'utf-8',
+                                      'invalid continuation byte',
+                                      s, pos, pos+2)
+                result.append(r)
                 continue
-            pos += 2
+            # 1110xxxx 10yyyyyy 10zzzzzz -> 00000000 xxxxyyyy yyzzzzzz
+            result.append(chr(ordch1))
+            result.append(chr(ordch2))
+            result.append(chr(ordch3))
+            pos += 3
 
-            # 1110xxxx 10yyyyyy 10zzzzzz -> 00000000 xxxxyyyy yyzzzzzz
-            res.append(chr(ordch1))
-            res.append(chr(ordch2))
-            res.append(chr(ordch3))
-            continue
-
-        if ordch1 <= 0xF4:
-            if (pos + 3) > end:
-                if not final:
-                    pos -= 1
-                    break
-                if pos < end and rutf8._invalid_byte_2_of_4(ordch1, ord(s[pos])):
-                    msg = "invalid continuation byte"
-                    r, pos, rettype = errorhandler(errors, "utf-8", msg, s,
-                                                   pos - 1, pos)
-                elif pos + 1 < end and rutf8._invalid_byte_3_of_4(ord(s[pos + 1])):
-                    msg = "invalid continuation byte"
-                    pos += 1
-                    r, pos, rettype = errorhandler(errors, "utf-8", msg, s,
-                                                   pos - 2, pos)
-                else:
-                    msg = "unexpected end of data"
-                    r, pos, rettype = errorhandler(errors, "utf-8", msg, s,
-                                                   pos - 1, pos)
-                    pos = end
-                res.append(r)
-                continue
-            ordch2 = ord(s[pos])
-            ordch3 = ord(s[pos + 1])
-            ordch4 = ord(s[pos + 2])
+        elif n == 4:
+            ordch2 = ord(s[pos+1])
+            ordch3 = ord(s[pos+2])
+            ordch4 = ord(s[pos+3])
             if rutf8._invalid_byte_2_of_4(ordch1, ordch2):
-                r, pos, rettype = errorhandler(errors, "utf-8", "invalid continuation byte",
-                    s, pos - 1, pos)
-                res.append(r)
+                r, pos, rettype = errorhandler(errors, 'utf-8',
+                                      'invalid continuation byte',
+                                      s, pos, pos+1)
+                result.append(r)
                 continue
             elif rutf8._invalid_byte_3_of_4(ordch3):
-                r, pos, rettype = errorhandler(errors, "utf-8", "invalid continuation byte",
-                    s, pos - 1, pos + 1)
-                res.append(r)
+                r, pos, rettype = errorhandler(errors, 'utf-8',
+                                      'invalid continuation byte',
+                                      s, pos, pos+2)
+                result.append(r)
                 continue
             elif rutf8._invalid_byte_4_of_4(ordch4):
-                r, pos, rettype = errorhandler(errors, "utf-8", "invalid continuation byte",
-                    s, pos - 1, pos + 2)
-                res.append(r)
+                r, pos, rettype = errorhandler(errors, 'utf-8',
+                                      'invalid continuation byte',
+                                      s, pos, pos+3)
+                result.append(r)
                 continue
+            # 11110www 10xxxxxx 10yyyyyy 10zzzzzz -> 000wwwxx xxxxyyyy yyzzzzzz
+            result.append(chr(ordch1))
+            result.append(chr(ordch2))
+            result.append(chr(ordch3))
+            result.append(chr(ordch4))
+            pos += 4
 
-            pos += 3
-            # 11110www 10xxxxxx 10yyyyyy 10zzzzzz -> 000wwwxx xxxxyyyy yyzzzzzz
-            res.append(chr(ordch1))
-            res.append(chr(ordch2))
-            res.append(chr(ordch3))
-            res.append(chr(ordch4))
-            continue
-
-        r, pos, rettype = errorhandler(errors, "utf-8", "invalid start byte",
-                s, pos - 1, pos)
-        res.append(r)
-
-    r = res.build()
+    r = result.build()
     return r, rutf8.check_utf8(r, True), pos
 
 hexdigits = "0123456789ABCDEFabcdef"
diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -1447,3 +1447,17 @@
         assert res == 52
         raises(TypeError, u"abc".encode, "test.mynontextenc")
         raises(TypeError, b"abc".decode, "test.mynontextenc")
+
+    def test_last_byte_handler(self):
+        # issue bb-2389
+        import _codecs
+        _codecs.register_error('custom_replace', lambda exc: (u'\ufffd', exc.start+1))
+        for s, res in ((b"WORD\xe3\xab",
+                            (u'WORD\ufffd\ufffd', u'WORD\ufffd')),
+                       (b"\xef\xbb\xbfWORD\xe3\xabWORD2",
+                            (u'\ufeffWORD\ufffd\ufffdWORD2',
+                             u'\ufeffWORD\ufffdWORD2'))):
+            r = s.decode('utf8', 'replace')
+            assert r == res[1]
+            r = s.decode('utf8', 'custom_replace')
+            assert r == res[0]
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -66,6 +66,12 @@
     from rpython.rlib.rgc import increase_root_stack_depth
     if new_limit <= 0:
         raise oefmt(space.w_ValueError, "recursion limit must be positive")
+    # Some programs use very large values to mean "don't check, I want to
+    # use as much as possible and then segfault".  Add a silent upper bound
+    # of 10**6 here, because huge values cause huge shadowstacks to be
+    # allocated (or MemoryErrors).
+    if new_limit > 1000000:
+        new_limit = 1000000
     try:
         _stack_set_length_fraction(new_limit * 0.001)
         _stack_check_noinline()
diff --git a/pypy/objspace/std/test/test_newformat.py b/pypy/objspace/std/test/test_newformat.py
--- a/pypy/objspace/std/test/test_newformat.py
+++ b/pypy/objspace/std/test/test_newformat.py
@@ -210,6 +210,13 @@
         fmtstr = self.s("{:[XYZ}")
         assert fmtstr.format(Foo()) == "<<%r>>" % (self.s("[XYZ"),)
 
+    def test_issue3100(self):
+        class Foo:
+            def __format__(self, f):
+                return '<<%r>>' % (f,)
+        fmtstr = self.s("{:[XYZ}")
+        assert fmtstr.format(Foo()) == "<<%r>>" % (self.s("[XYZ"),)
+
 
 class AppTestUnicodeFormat(BaseStringFormatTests):
     def setup_class(cls):

From pypy.commits at gmail.com  Sun Oct 27 06:12:01 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 27 Oct 2019 03:12:01 -0700 (PDT)
Subject: [pypy-commit] pypy default: socket -> _socket in tests
Message-ID: <5db56d71.1c69fb81.e7139.e6e8@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r97863:eba4da103b3e
Date: 2019-10-27 10:51 +0200
http://bitbucket.org/pypy/pypy/changeset/eba4da103b3e/

Log:	socket -> _socket in tests

diff --git a/pypy/module/select/test/test_select.py b/pypy/module/select/test/test_select.py
--- a/pypy/module/select/test/test_select.py
+++ b/pypy/module/select/test/test_select.py
@@ -338,10 +338,10 @@
         py.test.skip("build bot for s390x cannot open sockets")
 
     def w_make_server(self):
-        import socket
+        import _socket
         if hasattr(self, 'sock'):
             return self.sock
-        self.sock = socket.socket()
+        self.sock = _socket.socket()
         try_ports = [1023] + list(range(20000, 30000, 437))
         for port in try_ports:
             print('binding to port %d:' % (port,))
@@ -349,25 +349,25 @@
             try:
                 self.sock.bind(self.sockaddress)
                 break
-            except socket.error as e:   # should get a "Permission denied"
+            except _socket.error as e:   # should get a "Permission denied"
                 print(e)
             else:
                 raise(e)
 
     def w_getpair(self):
         """Helper method which returns a pair of connected sockets."""
-        import socket
+        import _socket
         import thread
 
         self.make_server()
 
         self.sock.listen(1)
-        s2 = socket.socket()
+        s2 = _socket.socket()
         thread.start_new_thread(s2.connect, (self.sockaddress,))
         s1, addr2 = self.sock.accept()
 
         # speed up the tests that want to fill the buffers
-        s1.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 4096)
-        s2.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 4096)
+        s1.setsockopt(_socket.SOL_SOCKET, _socket.SO_RCVBUF, 4096)
+        s2.setsockopt(_socket.SOL_SOCKET, _socket.SO_SNDBUF, 4096)
 
         return s1, s2

From pypy.commits at gmail.com  Sun Oct 27 06:12:03 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 27 Oct 2019 03:12:03 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: fix translation
Message-ID: <5db56d73.1c69fb81.d951d.9fdf@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r97864:bc23468af49e
Date: 2019-10-27 11:02 +0200
http://bitbucket.org/pypy/pypy/changeset/bc23468af49e/

Log:	fix translation

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -24,6 +24,7 @@
 
 def decode_never_raise(errors, encoding, msg, s, startingpos, endingpos):
     assert startingpos >= 0
+    assert endingpos >= 0
     ux = ['\ux' + hex(ord(x))[2:].upper() for x in s[startingpos:endingpos]]
     return ''.join(ux), endingpos, 'b'
 

From pypy.commits at gmail.com  Sun Oct 27 06:12:04 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 27 Oct 2019 03:12:04 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: move test_asyncio.py to extra_tests
Message-ID: <5db56d74.1c69fb81.ceb09.29a3@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r97865:bca10398bf62
Date: 2019-10-27 11:21 +0200
http://bitbucket.org/pypy/pypy/changeset/bca10398bf62/

Log:	move test_asyncio.py to extra_tests

diff --git a/pypy/module/_asyncio/test/test_asyncio.py b/extra_tests/test_asyncio.py
rename from pypy/module/_asyncio/test/test_asyncio.py
rename to extra_tests/test_asyncio.py
--- a/pypy/module/_asyncio/test/test_asyncio.py
+++ b/extra_tests/test_asyncio.py
@@ -6,7 +6,7 @@
     spaceconfig = dict(usemodules=["select","_socket","thread","signal",
                                    "struct","_multiprocessing","array",
                                    "_posixsubprocess",
-                                   "unicodedata"])
+                                   "unicodedata", "_asyncio"])
     if sys.platform == 'win32':
         pass
     else:
@@ -17,13 +17,13 @@
         # after calling run_until_complete
         """
 import encodings.idna
-import asyncio
+import _asyncio
 
 async def f():
-    reader, writer = await asyncio.open_connection('example.com', 80)
+    reader, writer = await _asyncio.open_connection('example.com', 80)
     writer.close()
 
-loop = asyncio.get_event_loop()
+loop = _asyncio.get_event_loop()
 loop.run_until_complete(f())
         """
     
@@ -32,7 +32,7 @@
         # and if the correct methods __aiter__ and __anext__ get called
         # and if the end results of run_until_complete are None (in a tuple)
         """
-import asyncio
+import _asyncio
 
 class AsyncIter:
     def __init__(self):
@@ -44,7 +44,7 @@
     
     async def __anext__(self):
         while self._index < 5:
-            await asyncio.sleep(1)
+            await _asyncio.sleep(1)
             self._index += 1
             return self._data[self._index-1]
         raise StopAsyncIteration
@@ -59,9 +59,9 @@
             self.res += "-"
 
 cor = Corotest()
-loop = asyncio.get_event_loop()
-futures = [asyncio.ensure_future(cor.do_loop()), asyncio.ensure_future(cor.do_loop())]
-taskres = loop.run_until_complete(asyncio.wait(futures))
+loop = _asyncio.get_event_loop()
+futures = [_asyncio.ensure_future(cor.do_loop()), _asyncio.ensure_future(cor.do_loop())]
+taskres = loop.run_until_complete(_asyncio.wait(futures))
 assert cor.res.count('0') == 2
 assert cor.res.count('1') == 2
 assert cor.res.count('2') == 2
@@ -83,7 +83,7 @@
         # hold the lock at the same time
         """
 import encodings.idna
-import asyncio
+import _asyncio
 
 class Corotest(object):
     def __init__(self):
@@ -93,13 +93,13 @@
         self.res += ' coro {}: waiting for lock -'.format(name)
         async with lock:
             self.res += ' coro {}: holding the lock -'.format(name)
-            await asyncio.sleep(1)
+            await _asyncio.sleep(1)
             self.res += ' coro {}: releasing the lock -'.format(name)
 
 cor = Corotest()
-loop = asyncio.get_event_loop()
-lock = asyncio.Lock()
-coros = asyncio.gather(cor.coro(1, lock), cor.coro(2, lock))
+loop = _asyncio.get_event_loop()
+lock = _asyncio.Lock()
+coros = _asyncio.gather(cor.coro(1, lock), cor.coro(2, lock))
 try:
     loop.run_until_complete(coros)
 finally:

From pypy.commits at gmail.com  Sun Oct 27 06:12:07 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 27 Oct 2019 03:12:07 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: redo for non-interplevel test
Message-ID: <5db56d77.1c69fb81.fb1f0.36b8@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r97866:49c202710476
Date: 2019-10-27 11:26 +0200
http://bitbucket.org/pypy/pypy/changeset/49c202710476/

Log:	redo for non-interplevel test

diff --git a/extra_tests/test_asyncio.py b/extra_tests/test_asyncio.py
--- a/extra_tests/test_asyncio.py
+++ b/extra_tests/test_asyncio.py
@@ -1,116 +1,96 @@
 import sys
 
-class AppTestAsyncIO(object):
-    """These tests are based on the async-await syntax of Python 3.5."""
+import encodings.idna
+import asyncio
+
+def test_async_gil_issue():
+    async def f():
+        reader, writer = await asyncio.open_connection('example.com', 80)
+        writer.close()
+
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(f())
     
-    spaceconfig = dict(usemodules=["select","_socket","thread","signal",
-                                   "struct","_multiprocessing","array",
-                                   "_posixsubprocess",
-                                   "unicodedata", "_asyncio"])
-    if sys.platform == 'win32':
-        pass
-    else:
-        spaceconfig['usemodules'].append('fcntl')
+def test_async_for():
+    # tests if async for receives all stores values in the right order
+    # and if the correct methods __aiter__ and __anext__ get called
+    # and if the end results of run_until_complete are None (in a tuple)
+    import asyncio
+
+    class AsyncIter:
+        def __init__(self):
+            self._data = list(range(5))
+            self._index = 0
+        
+        async def __aiter__(self):
+            return self
+        
+        async def __anext__(self):
+            while self._index < 5:
+                await asyncio.sleep(1)
+                self._index += 1
+                return self._data[self._index-1]
+            raise StopAsyncIteration
+
+    class Corotest(object):
+        def __init__(self):
+            self.res = "-"
+        
+        async def do_loop(self):
+            async for x in AsyncIter():
+                self.res += str(x)
+                self.res += "-"
+
+    cor = Corotest()
+    loop = asyncio.get_event_loop()
+    futures = [asyncio.ensure_future(cor.do_loop()), asyncio.ensure_future(cor.do_loop())]
+    taskres = loop.run_until_complete(asyncio.wait(futures))
+    assert cor.res.count('0') == 2
+    assert cor.res.count('1') == 2
+    assert cor.res.count('2') == 2
+    assert cor.res.count('3') == 2
+    assert cor.res.count('4') == 2
+    assert cor.res.find("0") < cor.res.find("1")
+    assert cor.res.find("1") < cor.res.find("2")
+    assert cor.res.find("2") < cor.res.find("3")
+    assert cor.res.find("3") < cor.res.find("4")
+    assert isinstance(taskres, tuple)
+    assert len(taskres) == 2
+    assert "result=None" in repr(taskres[0].pop())
+    assert "result=None" in repr(taskres[0].pop())
     
-    def test_gil_issue(self):
-        # the problem occured at await asyncio.open_connection
-        # after calling run_until_complete
-        """
-import encodings.idna
-import _asyncio
+def test_asynchronous_context_managers():
+    # it is important that "releasing lock A" happens before "holding lock B"
+    # or the other way around, but it is not allowed that both coroutines
+    # hold the lock at the same time
+    import encodings.idna
+    import asyncio
 
-async def f():
-    reader, writer = await _asyncio.open_connection('example.com', 80)
-    writer.close()
+    class Corotest(object):
+        def __init__(self):
+            self.res = "-"
+        
+        async def coro(self, name, lock):
+            self.res += ' coro {}: waiting for lock -'.format(name)
+            async with lock:
+                self.res += ' coro {}: holding the lock -'.format(name)
+                await asyncio.sleep(1)
+                self.res += ' coro {}: releasing the lock -'.format(name)
 
-loop = _asyncio.get_event_loop()
-loop.run_until_complete(f())
-        """
-    
-    def test_async_for(self):
-        # tests if async for receives all stores values in the right order
-        # and if the correct methods __aiter__ and __anext__ get called
-        # and if the end results of run_until_complete are None (in a tuple)
-        """
-import _asyncio
+    cor = Corotest()
+    loop = asyncio.get_event_loop()
+    lock = asyncio.Lock()
+    coros = asyncio.gather(cor.coro(1, lock), cor.coro(2, lock))
+    try:
+        loop.run_until_complete(coros)
+    finally:
+        loop.close()
 
-class AsyncIter:
-    def __init__(self):
-        self._data = list(range(5))
-        self._index = 0
-    
-    async def __aiter__(self):
-        return self
-    
-    async def __anext__(self):
-        while self._index < 5:
-            await _asyncio.sleep(1)
-            self._index += 1
-            return self._data[self._index-1]
-        raise StopAsyncIteration
-
-class Corotest(object):
-    def __init__(self):
-        self.res = "-"
-    
-    async def do_loop(self):
-        async for x in AsyncIter():
-            self.res += str(x)
-            self.res += "-"
-
-cor = Corotest()
-loop = _asyncio.get_event_loop()
-futures = [_asyncio.ensure_future(cor.do_loop()), _asyncio.ensure_future(cor.do_loop())]
-taskres = loop.run_until_complete(_asyncio.wait(futures))
-assert cor.res.count('0') == 2
-assert cor.res.count('1') == 2
-assert cor.res.count('2') == 2
-assert cor.res.count('3') == 2
-assert cor.res.count('4') == 2
-assert cor.res.find("0") < cor.res.find("1")
-assert cor.res.find("1") < cor.res.find("2")
-assert cor.res.find("2") < cor.res.find("3")
-assert cor.res.find("3") < cor.res.find("4")
-assert isinstance(taskres, tuple)
-assert len(taskres) == 2
-assert "result=None" in repr(taskres[0].pop())
-assert "result=None" in repr(taskres[0].pop())
-        """
-    
-    def test_asynchronous_context_managers(self):
-        # it is important that "releasing lock A" happens before "holding lock B"
-        # or the other way around, but it is not allowed that both coroutines
-        # hold the lock at the same time
-        """
-import encodings.idna
-import _asyncio
-
-class Corotest(object):
-    def __init__(self):
-        self.res = "-"
-    
-    async def coro(self, name, lock):
-        self.res += ' coro {}: waiting for lock -'.format(name)
-        async with lock:
-            self.res += ' coro {}: holding the lock -'.format(name)
-            await _asyncio.sleep(1)
-            self.res += ' coro {}: releasing the lock -'.format(name)
-
-cor = Corotest()
-loop = _asyncio.get_event_loop()
-lock = _asyncio.Lock()
-coros = _asyncio.gather(cor.coro(1, lock), cor.coro(2, lock))
-try:
-    loop.run_until_complete(coros)
-finally:
-    loop.close()
-
-assert "coro 1: waiting for lock" in cor.res
-assert "coro 1: holding the lock" in cor.res
-assert "coro 1: releasing the lock" in cor.res
-assert "coro 2: waiting for lock" in cor.res
-assert "coro 2: holding the lock" in cor.res
-assert "coro 2: releasing the lock" in cor.res
-assert cor.res.find("coro 1: releasing the lock") < cor.res.find("coro 2: holding the lock") or \
-cor.res.find("coro 2: releasing the lock") < cor.res.find("coro 1: holding the lock")
-        """
+    assert "coro 1: waiting for lock" in cor.res
+    assert "coro 1: holding the lock" in cor.res
+    assert "coro 1: releasing the lock" in cor.res
+    assert "coro 2: waiting for lock" in cor.res
+    assert "coro 2: holding the lock" in cor.res
+    assert "coro 2: releasing the lock" in cor.res
+    assert cor.res.find("coro 1: releasing the lock") < cor.res.find("coro 2: holding the lock") or \
+    cor.res.find("coro 2: releasing the lock") < cor.res.find("coro 1: holding the lock")

From pypy.commits at gmail.com  Sun Oct 27 06:12:08 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 27 Oct 2019 03:12:08 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: merge default into py3.6
Message-ID: <5db56d78.1c69fb81.243e3.b913@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r97867:189cdf258b08
Date: 2019-10-27 11:44 +0200
http://bitbucket.org/pypy/pypy/changeset/189cdf258b08/

Log:	merge default into py3.6

diff --git a/pypy/module/select/test/test_select.py b/pypy/module/select/test/test_select.py
--- a/pypy/module/select/test/test_select.py
+++ b/pypy/module/select/test/test_select.py
@@ -354,10 +354,10 @@
         py.test.skip("build bot for s390x cannot open sockets")
 
     def w_make_server(self):
-        import socket
+        import _socket
         if hasattr(self, 'sock'):
             return self.sock
-        self.sock = socket.socket()
+        self.sock = _socket.socket()
         try_ports = [1023] + list(range(20000, 30000, 437))
         for port in try_ports:
             print('binding to port %d:' % (port,))
@@ -365,14 +365,14 @@
             try:
                 self.sock.bind(self.sockaddress)
                 break
-            except socket.error as e:   # should get a "Permission denied"
+            except _socket.error as e:   # should get a "Permission denied"
                 print(e)
             else:
                 raise(e)
 
     def w_getpair(self):
         """Helper method which returns a pair of connected sockets."""
-        import socket
+        import _socket
         import _thread
 
         self.make_server()
@@ -380,12 +380,12 @@
         self.make_server()
 
         self.sock.listen(1)
-        s2 = socket.socket()
+        s2 = _socket.socket()
         _thread.start_new_thread(s2.connect, (self.sockaddress,))
         s1, addr2 = self.sock.accept()
 
         # speed up the tests that want to fill the buffers
-        s1.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 4096)
-        s2.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 4096)
+        s1.setsockopt(_socket.SOL_SOCKET, _socket.SO_RCVBUF, 4096)
+        s2.setsockopt(_socket.SOL_SOCKET, _socket.SO_SNDBUF, 4096)
 
         return s1, s2

From pypy.commits at gmail.com  Sun Oct 27 06:12:10 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 27 Oct 2019 03:12:10 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: fix for python3,
 _socket.accept -> _socket._accept
Message-ID: <5db56d7a.1c69fb81.2a802.86ac@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r97868:10516be1003b
Date: 2019-10-27 12:10 +0200
http://bitbucket.org/pypy/pypy/changeset/10516be1003b/

Log:	fix for python3, _socket.accept -> _socket._accept

diff --git a/pypy/module/select/test/test_select.py b/pypy/module/select/test/test_select.py
--- a/pypy/module/select/test/test_select.py
+++ b/pypy/module/select/test/test_select.py
@@ -382,7 +382,9 @@
         self.sock.listen(1)
         s2 = _socket.socket()
         _thread.start_new_thread(s2.connect, (self.sockaddress,))
-        s1, addr2 = self.sock.accept()
+        fd, addr2 = self.sock._accept()
+        s1 = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM,
+                              proto=0, fileno=fd)
 
         # speed up the tests that want to fill the buffers
         s1.setsockopt(_socket.SOL_SOCKET, _socket.SO_RCVBUF, 4096)

From pypy.commits at gmail.com  Sun Oct 27 14:33:37 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 27 Oct 2019 11:33:37 -0700 (PDT)
Subject: [pypy-commit] pypy default: backport ensurepip from 3.6
Message-ID: <5db5e301.1c69fb81.c347c.ba39@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r97869:17694ed47a34
Date: 2019-10-27 20:32 +0200
http://bitbucket.org/pypy/pypy/changeset/17694ed47a34/

Log:	backport ensurepip from 3.6

diff --git a/lib-python/2.7/ensurepip/__init__.py b/lib-python/2.7/ensurepip/__init__.py
--- a/lib-python/2.7/ensurepip/__init__.py
+++ b/lib-python/2.7/ensurepip/__init__.py
@@ -4,7 +4,6 @@
 import os
 import os.path
 import pkgutil
-import shutil
 import sys
 import tempfile
 
@@ -28,8 +27,8 @@
         sys.path = additional_paths + sys.path
 
     # Install the bundled software
-    import pip
-    pip.main(args)
+    import pip._internal
+    return pip._internal.main(args)
 
 
 def version():
@@ -38,7 +37,6 @@
     """
     return _PIP_VERSION
 
-
 def _disable_pip_configuration_settings():
     # We deliberately ignore all pip environment variables
     # when invoking pip
@@ -52,7 +50,7 @@
 
 
 def bootstrap(root=None, upgrade=False, user=False,
-              altinstall=False, default_pip=True,
+              altinstall=False, default_pip=False,
               verbosity=0):
     """
     Bootstrap pip into the current Python installation (or the given root
@@ -60,6 +58,21 @@
 
     Note that calling this function will alter both sys.path and os.environ.
     """
+    # Discard the return value
+    _bootstrap(root=root, upgrade=upgrade, user=user,
+               altinstall=altinstall, default_pip=default_pip,
+               verbosity=verbosity)
+
+
+def _bootstrap(root=None, upgrade=False, user=False,
+              altinstall=False, default_pip=False,
+              verbosity=0):
+    """
+    Bootstrap pip into the current Python installation (or the given root
+    directory). Returns pip command status code.
+
+    Note that calling this function will alter both sys.path and os.environ.
+    """
     if altinstall and default_pip:
         raise ValueError("Cannot use altinstall and default_pip together")
 
@@ -78,8 +91,7 @@
         # omit pip and easy_install
         os.environ["ENSUREPIP_OPTIONS"] = "install"
 
-    tmpdir = tempfile.mkdtemp()
-    try:
+    with tempfile.TemporaryDirectory() as tmpdir:
         # Put our bundled wheels into a temporary directory and construct the
         # additional paths that need added to sys.path
         additional_paths = []
@@ -105,10 +117,7 @@
         if verbosity:
             args += ["-" + "v" * verbosity]
 
-        _run_pip(args + [p[0] for p in _PROJECTS], additional_paths)
-    finally:
-        shutil.rmtree(tmpdir, ignore_errors=True)
-
+        return _run_pip(args + [p[0] for p in _PROJECTS], additional_paths)
 
 def _uninstall_helper(verbosity=0):
     """Helper to support a clean default uninstall process on Windows
@@ -135,7 +144,7 @@
     if verbosity:
         args += ["-" + "v" * verbosity]
 
-    _run_pip(args + [p[0] for p in reversed(_PROJECTS)])
+    return _run_pip(args + [p[0] for p in reversed(_PROJECTS)])
 
 
 def _main(argv=None):
@@ -176,27 +185,20 @@
         "--altinstall",
         action="store_true",
         default=False,
-        help=("Make an alternate install, installing only the X.Y versioned"
-              "scripts (Default: pipX, pipX.Y, easy_install-X.Y)"),
+        help=("Make an alternate install, installing only the X.Y versioned "
+              "scripts (Default: pipX, pipX.Y, easy_install-X.Y)."),
     )
     parser.add_argument(
         "--default-pip",
         action="store_true",
-        default=True,
-        dest="default_pip",
-        help=argparse.SUPPRESS,
-    )
-    parser.add_argument(
-        "--no-default-pip",
-        action="store_false",
-        dest="default_pip",
-        help=("Make a non default install, installing only the X and X.Y "
-              "versioned scripts."),
+        default=False,
+        help=("Make a default pip install, installing the unqualified pip "
+              "and easy_install in addition to the versioned scripts."),
     )
 
     args = parser.parse_args(argv)
 
-    bootstrap(
+    return _bootstrap(
         root=args.root,
         upgrade=args.upgrade,
         user=args.user,

From pypy.commits at gmail.com  Mon Oct 28 03:28:43 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 28 Oct 2019 00:28:43 -0700 (PDT)
Subject: [pypy-commit] pypy default: back out most of 17694ed47a34,
 leave only what is needed for pip 19.2 to work
Message-ID: <5db698ab.1c69fb81.19ba4.1699@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r97870:e159b221303b
Date: 2019-10-28 09:26 +0200
http://bitbucket.org/pypy/pypy/changeset/e159b221303b/

Log:	back out most of 17694ed47a34, leave only what is needed for pip
	19.2 to work

diff --git a/lib-python/2.7/ensurepip/__init__.py b/lib-python/2.7/ensurepip/__init__.py
--- a/lib-python/2.7/ensurepip/__init__.py
+++ b/lib-python/2.7/ensurepip/__init__.py
@@ -4,6 +4,7 @@
 import os
 import os.path
 import pkgutil
+import shutil
 import sys
 import tempfile
 
@@ -37,6 +38,7 @@
     """
     return _PIP_VERSION
 
+
 def _disable_pip_configuration_settings():
     # We deliberately ignore all pip environment variables
     # when invoking pip
@@ -50,7 +52,7 @@
 
 
 def bootstrap(root=None, upgrade=False, user=False,
-              altinstall=False, default_pip=False,
+              altinstall=False, default_pip=True,
               verbosity=0):
     """
     Bootstrap pip into the current Python installation (or the given root
@@ -58,21 +60,6 @@
 
     Note that calling this function will alter both sys.path and os.environ.
     """
-    # Discard the return value
-    _bootstrap(root=root, upgrade=upgrade, user=user,
-               altinstall=altinstall, default_pip=default_pip,
-               verbosity=verbosity)
-
-
-def _bootstrap(root=None, upgrade=False, user=False,
-              altinstall=False, default_pip=False,
-              verbosity=0):
-    """
-    Bootstrap pip into the current Python installation (or the given root
-    directory). Returns pip command status code.
-
-    Note that calling this function will alter both sys.path and os.environ.
-    """
     if altinstall and default_pip:
         raise ValueError("Cannot use altinstall and default_pip together")
 
@@ -91,7 +78,8 @@
         # omit pip and easy_install
         os.environ["ENSUREPIP_OPTIONS"] = "install"
 
-    with tempfile.TemporaryDirectory() as tmpdir:
+    tmpdir = tempfile.mkdtemp()
+    try:
         # Put our bundled wheels into a temporary directory and construct the
         # additional paths that need added to sys.path
         additional_paths = []
@@ -117,7 +105,10 @@
         if verbosity:
             args += ["-" + "v" * verbosity]
 
-        return _run_pip(args + [p[0] for p in _PROJECTS], additional_paths)
+        _run_pip(args + [p[0] for p in _PROJECTS], additional_paths)
+    finally:
+        shutil.rmtree(tmpdir, ignore_errors=True)
+
 
 def _uninstall_helper(verbosity=0):
     """Helper to support a clean default uninstall process on Windows
@@ -144,7 +135,7 @@
     if verbosity:
         args += ["-" + "v" * verbosity]
 
-    return _run_pip(args + [p[0] for p in reversed(_PROJECTS)])
+    _run_pip(args + [p[0] for p in reversed(_PROJECTS)])
 
 
 def _main(argv=None):
@@ -185,20 +176,27 @@
         "--altinstall",
         action="store_true",
         default=False,
-        help=("Make an alternate install, installing only the X.Y versioned "
-              "scripts (Default: pipX, pipX.Y, easy_install-X.Y)."),
+        help=("Make an alternate install, installing only the X.Y versioned"
+              "scripts (Default: pipX, pipX.Y, easy_install-X.Y)"),
     )
     parser.add_argument(
         "--default-pip",
         action="store_true",
-        default=False,
-        help=("Make a default pip install, installing the unqualified pip "
-              "and easy_install in addition to the versioned scripts."),
+        default=True,
+        dest="default_pip",
+        help=argparse.SUPPRESS,
+    )
+    parser.add_argument(
+        "--no-default-pip",
+        action="store_false",
+        dest="default_pip",
+        help=("Make a non default install, installing only the X and X.Y "
+              "versioned scripts."),
     )
 
     args = parser.parse_args(argv)
 
-    return _bootstrap(
+    bootstrap(
         root=args.root,
         upgrade=args.upgrade,
         user=args.user,

From pypy.commits at gmail.com  Mon Oct 28 05:47:22 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 28 Oct 2019 02:47:22 -0700 (PDT)
Subject: [pypy-commit] pypy default: set minimal MACOSX_DEPLOYMENT_TARGET to
 10.7 on macOS; cpython uses 10.5
Message-ID: <5db6b92a.1c69fb81.d05bb.671f@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r97871:158784f4a75c
Date: 2019-10-28 11:41 +0200
http://bitbucket.org/pypy/pypy/changeset/158784f4a75c/

Log:	set minimal MACOSX_DEPLOYMENT_TARGET to 10.7 on macOS; cpython uses
	10.5

diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -86,7 +86,7 @@
             arch = platform.machine()
         g['LDSHARED'] += ' -undefined dynamic_lookup'
         g['CC'] += ' -arch %s' % (arch,)
-        g['MACOSX_DEPLOYMENT_TARGET'] = '10.14'
+        g['MACOSX_DEPLOYMENT_TARGET'] = '10.7'
 
     global _config_vars
     _config_vars = g

From pypy.commits at gmail.com  Mon Oct 28 05:47:24 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 28 Oct 2019 02:47:24 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: set minimal MACOSX_DEPLOYMENT_TARGET to
 10.7 on macOS; cpython does not have this
Message-ID: <5db6b92c.1c69fb81.efac1.f727@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r97872:73d1f8c0a863
Date: 2019-10-28 11:46 +0200
http://bitbucket.org/pypy/pypy/changeset/73d1f8c0a863/

Log:	set minimal MACOSX_DEPLOYMENT_TARGET to 10.7 on macOS; cpython does
	not have this

diff --git a/lib-python/3/_osx_support.py b/lib-python/3/_osx_support.py
--- a/lib-python/3/_osx_support.py
+++ b/lib-python/3/_osx_support.py
@@ -109,7 +109,7 @@
             # else: fall back to the default behaviour
     if not _SYSTEM_VERSION:
         # minimum supported MACOSX_DEPLOYMENT_TARGET version
-        return '10.14'
+        return '10.7'
     return _SYSTEM_VERSION
 
 def _remove_original_values(_config_vars):

From pypy.commits at gmail.com  Mon Oct 28 06:54:56 2019
From: pypy.commits at gmail.com (arigo)
Date: Mon, 28 Oct 2019 03:54:56 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: Issue #2687
Message-ID: <5db6c900.1c69fb81.a7972.4e59@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.6
Changeset: r97873:e7f5d75f3f87
Date: 2019-10-28 11:54 +0100
http://bitbucket.org/pypy/pypy/changeset/e7f5d75f3f87/

Log:	Issue #2687

	Implement pystrhex.h

diff --git a/pypy/module/cpyext/include/pystrhex.h b/pypy/module/cpyext/include/pystrhex.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/include/pystrhex.h
@@ -0,0 +1,1 @@
+/* empty */
diff --git a/pypy/module/cpyext/moduledef.py b/pypy/module/cpyext/moduledef.py
--- a/pypy/module/cpyext/moduledef.py
+++ b/pypy/module/cpyext/moduledef.py
@@ -79,6 +79,7 @@
 import pypy.module.cpyext.marshal
 import pypy.module.cpyext.genobject
 import pypy.module.cpyext.namespaceobject
+import pypy.module.cpyext.pystrhex
 
 # now that all rffi_platform.Struct types are registered, configure them
 api.configure_types()
diff --git a/pypy/module/cpyext/pystrhex.py b/pypy/module/cpyext/pystrhex.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/pystrhex.py
@@ -0,0 +1,26 @@
+import sys
+from pypy.module.cpyext.api import cts
+from rpython.rlib.rstring import StringBuilder
+
+ at cts.decl("PyObject * _Py_strhex(const char* argbuf, const Py_ssize_t arglen)")
+def _Py_strhex(space, argbuf, arglen):
+    s = strhex(argbuf, arglen)
+    return space.newutf8(s, len(s))    # contains ascii only
+
+ at cts.decl("PyObject * _Py_strhex_bytes(const char* argbuf, const Py_ssize_t arglen)")
+def _Py_strhex_bytes(space, argbuf, arglen):
+    s = strhex(argbuf, arglen)
+    return space.newbytes(s)
+
+hexdigits = "0123456789abcdef"
+
+def strhex(argbuf, arglen):
+    assert arglen >= 0
+    if arglen > sys.maxint / 2:
+        raise MemoryError
+    builder = StringBuilder(arglen * 2)
+    for i in range(arglen):
+        b = ord(argbuf[i])
+        builder.append(hexdigits[(b >> 4) & 0xf])
+        builder.append(hexdigits[b & 0xf])
+    return builder.build()
diff --git a/pypy/module/cpyext/test/test_pystrhex.py b/pypy/module/cpyext/test/test_pystrhex.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/test_pystrhex.py
@@ -0,0 +1,20 @@
+from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+
+class AppTestPyStrHex(AppTestCpythonExtensionBase):
+    def test_simple(self):
+        module = self.import_extension('strhex', [
+            ("new", "METH_NOARGS",
+             """
+                 static char a[] = { (char)255, 0, 65 };
+                 PyObject *o1 = _Py_strhex(a, 3);
+                 PyObject *o2 = _Py_strhex_bytes(a, 3);
+                 return PyTuple_Pack(2, o1, o2);
+             """)],
+            prologue="""
+                 #include <pystrhex.h>
+            """)
+        o1, o2 = module.new()
+        assert type(o1) is str
+        assert type(o2) is bytes
+        assert o1 == u"ff0041"
+        assert o2 == b"ff0041"

From pypy.commits at gmail.com  Mon Oct 28 08:46:27 2019
From: pypy.commits at gmail.com (arigo)
Date: Mon, 28 Oct 2019 05:46:27 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix in the aarch64 backend for a rare
 case where a register would be overridden
Message-ID: <5db6e323.1c69fb81.bae94.d26c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r97874:81c30ab04ab4
Date: 2019-10-28 13:45 +0100
http://bitbucket.org/pypy/pypy/changeset/81c30ab04ab4/

Log:	Fix in the aarch64 backend for a rare case where a register would be
	overridden

diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py
--- a/rpython/jit/backend/aarch64/opassembler.py
+++ b/rpython/jit/backend/aarch64/opassembler.py
@@ -986,7 +986,8 @@
             loc_index = arglocs[1]
             assert loc_index.is_core_reg()
             tmp1 = r.ip1
-            tmp2 = arglocs[-1]  # the last item is a preallocated tmp
+            #tmp2 = arglocs[-1]  -- the last item is a preallocated tmp on arm,
+            #                       but not here on aarch64
             # lr = byteofs
             s = 3 + descr.jit_wb_card_page_shift
             mc.MVN_rr_shifted(r.lr.value, loc_index.value, s, shifttype=shift.LSR)
@@ -997,10 +998,10 @@
                             descr.jit_wb_card_page_shift, shifttype=shift.LSR)
 
             # set the bit
-            mc.MOVZ_r_u16(tmp2.value, 1, 0)
+            mc.MOVZ_r_u16(r.ip0.value, 1, 0)
+            mc.LSL_rr(tmp1.value, r.ip0.value, tmp1.value)
             mc.LDRB_rr(r.ip0.value, loc_base.value, r.lr.value)
-            mc.LSL_rr(tmp2.value, tmp2.value, tmp1.value)
-            mc.ORR_rr(r.ip0.value, r.ip0.value, tmp2.value)
+            mc.ORR_rr(r.ip0.value, r.ip0.value, tmp1.value)
             mc.STR_size_rr(0, r.ip0.value, loc_base.value, r.lr.value)
             # done
             #

From pypy.commits at gmail.com  Mon Oct 28 08:57:43 2019
From: pypy.commits at gmail.com (arigo)
Date: Mon, 28 Oct 2019 05:57:43 -0700 (PDT)
Subject: [pypy-commit] pypy default: Merged in
 danchr/pypy/allow-forcing-no-embed (pull request #676)
Message-ID: <5db6e5c7.1c69fb81.c86c2.226e@mx.google.com>

Author: Armin Rigo <armin.rigo at gmail.com>
Branch: 
Changeset: r97876:54854c13aaba
Date: 2019-10-28 12:56 +0000
http://bitbucket.org/pypy/pypy/changeset/54854c13aaba/

Log:	Merged in danchr/pypy/allow-forcing-no-embed (pull request #676)

	package: allow suppressing embedded dependencies with an envvar

diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -320,6 +320,8 @@
         options.no_tk = True
     if os.environ.has_key("PYPY_EMBED_DEPENDENCIES"):
         options.embed_dependencies = True
+    elif os.environ.has_key("PYPY_NO_EMBED_DEPENDENCIES"):
+        options.embed_dependencies = False
     if not options.builddir:
         # The import actually creates the udir directory
         from rpython.tool.udir import udir

From pypy.commits at gmail.com  Mon Oct 28 08:57:59 2019
From: pypy.commits at gmail.com (danchr)
Date: Mon, 28 Oct 2019 05:57:59 -0700 (PDT)
Subject: [pypy-commit] pypy allow-forcing-no-embed: package: allow
 suppressing embedded dependencies with an envvar
Message-ID: <5db6e5d7.1c69fb81.953ec.f1b3@mx.google.com>

Author: Dan Villiom Podlaski Christiansen <danchr at gmail.com>
Branch: allow-forcing-no-embed
Changeset: r97875:bf745505226d
Date: 2019-10-28 09:47 +0100
http://bitbucket.org/pypy/pypy/changeset/bf745505226d/

Log:	package: allow suppressing embedded dependencies with an envvar

diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -320,6 +320,8 @@
         options.no_tk = True
     if os.environ.has_key("PYPY_EMBED_DEPENDENCIES"):
         options.embed_dependencies = True
+    elif os.environ.has_key("PYPY_NO_EMBED_DEPENDENCIES"):
+        options.embed_dependencies = False
     if not options.builddir:
         # The import actually creates the udir directory
         from rpython.tool.udir import udir

From pypy.commits at gmail.com  Mon Oct 28 09:10:07 2019
From: pypy.commits at gmail.com (arigo)
Date: Mon, 28 Oct 2019 06:10:07 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: hg merge default
Message-ID: <5db6e8af.1c69fb81.aad9e.2bbc@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.6
Changeset: r97877:7487744729a8
Date: 2019-10-28 14:09 +0100
http://bitbucket.org/pypy/pypy/changeset/7487744729a8/

Log:	hg merge default

diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -329,6 +329,8 @@
         options.no_tk = True
     if os.environ.has_key("PYPY_EMBED_DEPENDENCIES"):
         options.embed_dependencies = True
+    elif os.environ.has_key("PYPY_NO_EMBED_DEPENDENCIES"):
+        options.embed_dependencies = False
     if not options.builddir:
         # The import actually creates the udir directory
         from rpython.tool.udir import udir
diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py
--- a/rpython/jit/backend/aarch64/opassembler.py
+++ b/rpython/jit/backend/aarch64/opassembler.py
@@ -986,7 +986,8 @@
             loc_index = arglocs[1]
             assert loc_index.is_core_reg()
             tmp1 = r.ip1
-            tmp2 = arglocs[-1]  # the last item is a preallocated tmp
+            #tmp2 = arglocs[-1]  -- the last item is a preallocated tmp on arm,
+            #                       but not here on aarch64
             # lr = byteofs
             s = 3 + descr.jit_wb_card_page_shift
             mc.MVN_rr_shifted(r.lr.value, loc_index.value, s, shifttype=shift.LSR)
@@ -997,10 +998,10 @@
                             descr.jit_wb_card_page_shift, shifttype=shift.LSR)
 
             # set the bit
-            mc.MOVZ_r_u16(tmp2.value, 1, 0)
+            mc.MOVZ_r_u16(r.ip0.value, 1, 0)
+            mc.LSL_rr(tmp1.value, r.ip0.value, tmp1.value)
             mc.LDRB_rr(r.ip0.value, loc_base.value, r.lr.value)
-            mc.LSL_rr(tmp2.value, tmp2.value, tmp1.value)
-            mc.ORR_rr(r.ip0.value, r.ip0.value, tmp2.value)
+            mc.ORR_rr(r.ip0.value, r.ip0.value, tmp1.value)
             mc.STR_size_rr(0, r.ip0.value, loc_base.value, r.lr.value)
             # done
             #

From pypy.commits at gmail.com  Mon Oct 28 14:30:53 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 28 Oct 2019 11:30:53 -0700 (PDT)
Subject: [pypy-commit] pypy allow-forcing-no-embed: close and document
 merged branch
Message-ID: <5db733dd.1c69fb81.d5e78.3454@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: allow-forcing-no-embed
Changeset: r97878:aad09c46bd5c
Date: 2019-10-28 20:29 +0200
http://bitbucket.org/pypy/pypy/changeset/aad09c46bd5c/

Log:	close and document merged branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -16,3 +16,7 @@
 .. branch: license-update
 
 Update list directories in LICENSE
+
+.. branch: allow-forcing-no-embed
+
+When packaging, allow suppressing embedded dependencies via PYPY_NO_EMBED_DEPENDENCIES

From pypy.commits at gmail.com  Mon Oct 28 14:30:55 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 28 Oct 2019 11:30:55 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge allow-forcing-no-embed which
 added PYPY_NO_EMBED_DEPENDENCIES to packaging
Message-ID: <5db733df.1c69fb81.787dc.1c58@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r97879:d6217bf98b7c
Date: 2019-10-28 20:29 +0200
http://bitbucket.org/pypy/pypy/changeset/d6217bf98b7c/

Log:	merge allow-forcing-no-embed which added PYPY_NO_EMBED_DEPENDENCIES
	to packaging

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -16,3 +16,7 @@
 .. branch: license-update
 
 Update list directories in LICENSE
+
+.. branch: allow-forcing-no-embed
+
+When packaging, allow suppressing embedded dependencies via PYPY_NO_EMBED_DEPENDENCIES

From pypy.commits at gmail.com  Tue Oct 29 07:38:20 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 29 Oct 2019 04:38:20 -0700 (PDT)
Subject: [pypy-commit] pypy default: move test that used to use _ssl to _demo
Message-ID: <5db824ac.1c69fb81.a88fd.f689@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r97880:037744e9abc7
Date: 2019-10-29 13:34 +0200
http://bitbucket.org/pypy/pypy/changeset/037744e9abc7/

Log:	move test that used to use _ssl to _demo

diff --git a/pypy/interpreter/test/test_appinterp.py b/pypy/interpreter/test/test_appinterp.py
--- a/pypy/interpreter/test/test_appinterp.py
+++ b/pypy/interpreter/test/test_appinterp.py
@@ -158,25 +158,3 @@
         w_str = space1.getattr(w_mymod1, space1.wrap("hi"))
         assert space1.str_w(w_str) == "hello"
 
-class TestMixedModuleUnfreeze:
-    spaceconfig = dict(usemodules=('_ssl', '_socket'))
-
-    def test_random_stuff_can_unfreeze(self):
-        # When a module contains an "import" statement in applevel code, the
-        # imported module is initialized, possibly after it has been already
-        # frozen.
-
-        # This is important when the module startup() function does something
-        # at runtime, like setting os.environ (posix module) or initializing
-        # the winsock library (_socket module)
-        w_socket = self.space.builtin_modules['_socket']
-        w_ssl = self.space.builtin_modules['_ssl']
-
-        # Uncomment this line for a workaround
-        # space.getattr(w_ssl, space.wrap('SSLError'))
-
-        w_socket._cleanup_()
-        assert w_socket.startup_called == False
-        w_ssl._cleanup_() # w_ssl.appleveldefs['SSLError'] imports _socket
-        assert w_socket.startup_called == False
-
diff --git a/pypy/module/_demo/app_demo.py b/pypy/module/_demo/app_demo.py
--- a/pypy/module/_demo/app_demo.py
+++ b/pypy/module/_demo/app_demo.py
@@ -1,3 +1,8 @@
+# import posix underneath
+from os import environ
+
+# for the test_random_stuff_can_unfreeze test
+environ['PYPY_DEMO_MODULE_ERROR'] = '1'
 
 class DemoError(Exception):
     pass
diff --git a/pypy/module/_demo/test/test_import.py b/pypy/module/_demo/test/test_import.py
--- a/pypy/module/_demo/test/test_import.py
+++ b/pypy/module/_demo/test/test_import.py
@@ -28,3 +28,23 @@
         assert Module.demo_events == ['setup', 'startup']
 
         assert space.getattr(w_demo, space.wrap('measuretime'))
+
+class TestMixedModuleUnfreeze:
+    spaceconfig = dict(usemodules=('_demo', 'posix'))
+
+    def test_random_stuff_can_unfreeze(self):
+        # When a module contains an "import" statement in applevel code, the
+        # imported module is initialized, possibly after it has been already
+        # frozen.
+
+        # This is important when the module startup() function does something
+        # at runtime, like setting os.environ (posix module) or initializing
+        # the winsock library (_socket module)
+        w_posix = self.space.builtin_modules['posix']
+        w_demo = self.space.builtin_modules['_demo']
+
+        w_posix._cleanup_()
+        assert w_posix.startup_called == False
+        w_demo._cleanup_() # w_demo.appleveldefs['DemoError'] imports posix
+        assert w_posix.startup_called == False
+

From pypy.commits at gmail.com  Tue Oct 29 07:38:22 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 29 Oct 2019 04:38:22 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: merge default into branch
Message-ID: <5db824ae.1c69fb81.a080.d1c1@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r97881:50cb48b9e6e5
Date: 2019-10-29 13:36 +0200
http://bitbucket.org/pypy/pypy/changeset/50cb48b9e6e5/

Log:	merge default into branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -16,3 +16,7 @@
 .. branch: license-update
 
 Update list directories in LICENSE
+
+.. branch: allow-forcing-no-embed
+
+When packaging, allow suppressing embedded dependencies via PYPY_NO_EMBED_DEPENDENCIES
diff --git a/pypy/interpreter/test/test_appinterp.py b/pypy/interpreter/test/test_appinterp.py
--- a/pypy/interpreter/test/test_appinterp.py
+++ b/pypy/interpreter/test/test_appinterp.py
@@ -158,25 +158,3 @@
         w_str = space1.getattr(w_mymod1, space1.wrap("hi"))
         assert space1.text_w(w_str) == "hello"
 
-class TestMixedModuleUnfreeze:
-    spaceconfig = dict(usemodules=('_socket',))
-
-    def test_random_stuff_can_unfreeze(self):
-        # When a module contains an "import" statement in applevel code, the
-        # imported module is initialized, possibly after it has been already
-        # frozen.
-
-        # This is important when the module startup() function does something
-        # at runtime, like setting os.environ (posix module) or initializing
-        # the winsock library (_socket module)
-        w_socket = self.space.builtin_modules['_socket']
-        # _ssl is not builtin anymore, this test also tried to _cleanup_ on
-        # the wrapped ssl object
-        # w_ssl = self.space.builtin_modules['_ssl']
-
-        # Uncomment this line for a workaround
-        # space.getattr(w_ssl, space.wrap('SSLError'))
-
-        w_socket._cleanup_()
-        assert w_socket.startup_called == False
-
diff --git a/pypy/module/_demo/app_demo.py b/pypy/module/_demo/app_demo.py
--- a/pypy/module/_demo/app_demo.py
+++ b/pypy/module/_demo/app_demo.py
@@ -1,3 +1,8 @@
+# import posix underneath
+from os import environ
+
+# for the test_random_stuff_can_unfreeze test
+environ['PYPY_DEMO_MODULE_ERROR'] = '1'
 
 class DemoError(Exception):
     pass
diff --git a/pypy/module/_demo/test/test_import.py b/pypy/module/_demo/test/test_import.py
--- a/pypy/module/_demo/test/test_import.py
+++ b/pypy/module/_demo/test/test_import.py
@@ -28,3 +28,23 @@
         assert Module.demo_events == ['setup', 'startup']
 
         assert space.getattr(w_demo, space.wrap('measuretime'))
+
+class TestMixedModuleUnfreeze:
+    spaceconfig = dict(usemodules=('_demo', 'posix'))
+
+    def test_random_stuff_can_unfreeze(self):
+        # When a module contains an "import" statement in applevel code, the
+        # imported module is initialized, possibly after it has been already
+        # frozen.
+
+        # This is important when the module startup() function does something
+        # at runtime, like setting os.environ (posix module) or initializing
+        # the winsock library (_socket module)
+        w_posix = self.space.builtin_modules['posix']
+        w_demo = self.space.builtin_modules['_demo']
+
+        w_posix._cleanup_()
+        assert w_posix.startup_called == False
+        w_demo._cleanup_() # w_demo.appleveldefs['DemoError'] imports posix
+        assert w_posix.startup_called == False
+

From pypy.commits at gmail.com  Tue Oct 29 09:41:27 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 29 Oct 2019 06:41:27 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: use an extension name like pypy36-pp73
 not pypy3-73 to prevent conflicts between py3.6, py.37
Message-ID: <5db84187.1c69fb81.954e3.ddd4@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r97882:c4659943384c
Date: 2019-10-29 15:39 +0200
http://bitbucket.org/pypy/pypy/changeset/c4659943384c/

Log:	use an extension name like pypy36-pp73 not pypy3-73 to prevent
	conflicts between py3.6, py.37

diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -21,8 +21,8 @@
 _WIN32 = sys.platform == 'win32'
 
 SO = '.pyd' if _WIN32 else '.so'
-PREFIX = 'pypy3-'
-DEFAULT_SOABI_BASE = '%s%d%d' % ((PREFIX,) + PYPY_VERSION[:2])
+PREFIX = 'pypy%d%d-' % CPYTHON_VERSION[:2]
+DEFAULT_SOABI_BASE = '%spp%d%d' % ((PREFIX,) + PYPY_VERSION[:2])
 
 PYC_TAG = '%s%d%d' % ((PREFIX,) + CPYTHON_VERSION[:2])   # 'pypy3-XY'
 
diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py
--- a/pypy/module/imp/test/test_app.py
+++ b/pypy/module/imp/test/test_app.py
@@ -315,4 +315,5 @@
         import sys
         if not hasattr(sys, 'pypy_version_info'):
             skip('This test is PyPy-only')
-        assert imp.get_tag() == 'pypy3-%d%d' % sys.version_info[0:2]
+        assert imp.get_tag() == 'pypy%d%d-pp%d%d' % (
+                            sys.version_info[:2] + sys.pypy_version_info[:2])

From pypy.commits at gmail.com  Tue Oct 29 09:41:29 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 29 Oct 2019 06:41:29 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: port some of the changes from macports
 pypy
Message-ID: <5db84189.1c69fb81.60266.6a87@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r97883:117749e404fd
Date: 2019-10-29 15:40 +0200
http://bitbucket.org/pypy/pypy/changeset/117749e404fd/

Log:	port some of the changes from macports pypy

	https://github.com/macports/macports-
	ports/blob/master/lang/pypy/files/darwin.py.diff

diff --git a/rpython/translator/platform/darwin.py b/rpython/translator/platform/darwin.py
--- a/rpython/translator/platform/darwin.py
+++ b/rpython/translator/platform/darwin.py
@@ -39,7 +39,7 @@
         # we get the basename of the executable we're trying to build.
         return (list(self.shared_only)
                 + ['-dynamiclib', '-install_name', '@rpath/' + target_basename,
-                   '-undefined', 'dynamic_lookup', '-flat_namespace']
+                   '-undefined', 'dynamic_lookup']
                 + args)
 
     def _include_dirs_for_libffi(self):
@@ -113,11 +113,15 @@
 
 class Darwin_PowerPC(Darwin):#xxx fixme, mwp
     name = "darwin_powerpc"
+    link_flags = Darwin.link_flags + ('-arch', 'ppc')
+    cflags = Darwin.cflags + ('-arch', 'ppc')
 
 class Darwin_i386(Darwin):
     name = "darwin_i386"
-    DEFAULT_CC = 'clang -arch i386'
+    link_flags = Darwin.link_flags + ('-arch', 'i386')
+    cflags = Darwin.cflags + ('-arch', 'i386')
 
 class Darwin_x86_64(Darwin):
     name = "darwin_x86_64"
-    DEFAULT_CC = 'clang -arch x86_64'
+    link_flags = Darwin.link_flags + ('-arch', 'x86_64')
+    cflags = Darwin.cflags + ('-arch', 'x86_64')

From pypy.commits at gmail.com  Tue Oct 29 12:02:51 2019
From: pypy.commits at gmail.com (arigo)
Date: Tue, 29 Oct 2019 09:02:51 -0700 (PDT)
Subject: [pypy-commit] pypy default: Import posix directly instead of os;
 import nt on win32
Message-ID: <5db862ab.1c69fb81.ec226.fef9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r97885:b50c4326c73f
Date: 2019-10-29 17:01 +0100
http://bitbucket.org/pypy/pypy/changeset/b50c4326c73f/

Log:	Import posix directly instead of os; import nt on win32

diff --git a/pypy/module/_demo/app_demo.py b/pypy/module/_demo/app_demo.py
--- a/pypy/module/_demo/app_demo.py
+++ b/pypy/module/_demo/app_demo.py
@@ -1,8 +1,12 @@
-# import posix underneath
-from os import environ
+import sys
+if sys.platform == 'win32':
+    import nt as posix
+else:
+    import posix
 
 # for the test_random_stuff_can_unfreeze test
-environ['PYPY_DEMO_MODULE_ERROR'] = '1'
+posix.environ['PYPY_DEMO_MODULE_ERROR'] = '1'
+
 
 class DemoError(Exception):
     pass
diff --git a/pypy/module/_demo/test/test_import.py b/pypy/module/_demo/test/test_import.py
--- a/pypy/module/_demo/test/test_import.py
+++ b/pypy/module/_demo/test/test_import.py
@@ -1,3 +1,4 @@
+import sys
 from pypy.module._demo.moduledef import Module
 from pypy.tool.option import make_config, make_objspace
 
@@ -29,8 +30,11 @@
 
         assert space.getattr(w_demo, space.wrap('measuretime'))
 
+
+posixname = 'posix' if sys.platform != 'win32' else 'nt'
+
 class TestMixedModuleUnfreeze:
-    spaceconfig = dict(usemodules=('_demo', 'posix'))
+    spaceconfig = dict(usemodules=('_demo', posixname))
 
     def test_random_stuff_can_unfreeze(self):
         # When a module contains an "import" statement in applevel code, the
@@ -40,7 +44,7 @@
         # This is important when the module startup() function does something
         # at runtime, like setting os.environ (posix module) or initializing
         # the winsock library (_socket module)
-        w_posix = self.space.builtin_modules['posix']
+        w_posix = self.space.builtin_modules[posixname]
         w_demo = self.space.builtin_modules['_demo']
 
         w_posix._cleanup_()

From pypy.commits at gmail.com  Tue Oct 29 14:42:14 2019
From: pypy.commits at gmail.com (arigo)
Date: Tue, 29 Oct 2019 11:42:14 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: Turn these flags,
 containing each 1 to 3 bits, from "unsigned int" to "unsigned
Message-ID: <5db88806.1c69fb81.9876c.57e7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.6
Changeset: r97886:1c74a232abde
Date: 2019-10-29 18:21 +0100
http://bitbucket.org/pypy/pypy/changeset/1c74a232abde/

Log:	Turn these flags, containing each 1 to 3 bits, from "unsigned int"
	to "unsigned char"

diff --git a/pypy/module/cpyext/parse/cpyext_unicodeobject.h b/pypy/module/cpyext/parse/cpyext_unicodeobject.h
--- a/pypy/module/cpyext/parse/cpyext_unicodeobject.h
+++ b/pypy/module/cpyext/parse/cpyext_unicodeobject.h
@@ -26,7 +26,7 @@
            If interned != SSTATE_NOT_INTERNED, the two references from the
            dictionary to this object are *not* counted in ob_refcnt.
          */
-        unsigned int interned;
+        unsigned char interned;
         /* Character size:
 
            - PyUnicode_WCHAR_KIND (0):
@@ -54,21 +54,21 @@
              * all characters are in the range U+0000-U+10FFFF
              * at least one character is in the range U+10000-U+10FFFF
          */
-        unsigned int kind;
+        unsigned char kind;
         /* Compact is with respect to the allocation scheme. Compact unicode
            objects only require one memory block while non-compact objects use
            one block for the PyUnicodeObject struct and another for its data
            buffer. */
-        unsigned int compact;
+        unsigned char compact;
         /* The string only contains characters in the range U+0000-U+007F (ASCII)
            and the kind is PyUnicode_1BYTE_KIND. If ascii is set and compact is
            set, use the PyASCIIObject structure. */
-        unsigned int ascii;
+        unsigned char ascii;
         /* The ready flag indicates whether the object layout is initialized
            completely. This means that this is either a compact object, or
            the data pointer is filled out. The bit is redundant, and helps
            to minimize the test in PyUnicode_IS_READY(). */
-        unsigned int ready;
+        unsigned char ready;
         /* Padding to ensure that PyUnicode_DATA() is always aligned to
            4 bytes (see issue #19537 on m68k). */
         /* not on PyPy */
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -124,19 +124,19 @@
     return rffi.getintfield(get_state(py_obj), 'c_kind')
 
 def set_kind(py_obj, value):
-    get_state(py_obj).c_kind = cts.cast('unsigned int', value)
+    get_state(py_obj).c_kind = cts.cast('unsigned char', value)
 
 def get_ascii(py_obj):
     return rffi.getintfield(get_state(py_obj), 'c_ascii')
 
 def set_ascii(py_obj, value):
-    get_state(py_obj).c_ascii = cts.cast('unsigned int', value)
+    get_state(py_obj).c_ascii = cts.cast('unsigned char', value)
 
 def get_ready(py_obj):
     return rffi.getintfield(get_state(py_obj), 'c_ready')
 
 def set_ready(py_obj, value):
-    get_state(py_obj).c_ready = cts.cast('unsigned int', value)
+    get_state(py_obj).c_ready = cts.cast('unsigned char', value)
 
 def get_wbuffer(py_obj):
     py_obj = cts.cast('PyASCIIObject*', py_obj)

From pypy.commits at gmail.com  Tue Oct 29 14:42:16 2019
From: pypy.commits at gmail.com (arigo)
Date: Tue, 29 Oct 2019 11:42:16 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: I *think* that's all we need to fix these
 XXX for sizeof(wchar_t)==2
Message-ID: <5db88808.1c69fb81.8c7a8.1d1b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.6
Changeset: r97887:98fbd0ef00cc
Date: 2019-10-29 19:08 +0100
http://bitbucket.org/pypy/pypy/changeset/98fbd0ef00cc/

Log:	I *think* that's all we need to fix these XXX for sizeof(wchar_t)==2

diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -313,10 +313,14 @@
             set_utf8(py_obj, cts.cast('char *', 0))
             set_utf8_len(py_obj, 0)
     elif maxchar < 65536:
-        # XXX: assumes that sizeof(wchar_t) == 4
         ucs2_str = utf8_encode_utf_16_helper(
             value, 'strict',
             byteorder=BYTEORDER)
+        if rffi.sizeof(lltype.UniChar) == 2 and not get_wbuffer(py_obj):
+            # Copy unicode buffer
+            wchar = cts.cast('wchar_t*', rffi.str2charp(ucs2_str))
+            set_wbuffer(py_obj, wchar)
+            set_wsize(py_obj, len(ucs2_str) // 2)
         ucs2_data = cts.cast('Py_UCS2 *', rffi.str2charp(ucs2_str))
         set_data(py_obj, cts.cast('void*', ucs2_data))
         set_len(py_obj, get_wsize(py_obj))
@@ -324,11 +328,10 @@
         set_utf8(py_obj, cts.cast('char *', 0))
         set_utf8_len(py_obj, 0)
     else:
-        # XXX: assumes that sizeof(wchar_t) == 4
         ucs4_str = utf8_encode_utf_32_helper(
             value, 'strict',
             byteorder=BYTEORDER)
-        if not get_wbuffer(py_obj):
+        if rffi.sizeof(lltype.UniChar) == 4 and not get_wbuffer(py_obj):
             # Copy unicode buffer
             wchar = cts.cast('wchar_t*', rffi.str2charp(ucs4_str))
             set_wbuffer(py_obj, wchar)

From pypy.commits at gmail.com  Tue Oct 29 15:21:47 2019
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 29 Oct 2019 12:21:47 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: fix test
Message-ID: <5db8914b.1c69fb81.8b73b.66ec@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6
Changeset: r97888:800619cae177
Date: 2019-10-29 19:20 +0000
http://bitbucket.org/pypy/pypy/changeset/800619cae177/

Log:	fix test

diff --git a/lib-python/3/test/test_bdb.py b/lib-python/3/test/test_bdb.py
--- a/lib-python/3/test/test_bdb.py
+++ b/lib-python/3/test/test_bdb.py
@@ -726,7 +726,7 @@
                 ('line', 2, 'tfunc_import'), ('step', ),
                 ('line', 3, 'tfunc_import'), ('quit', ),
             ]
-            skip = ('importlib*', TEST_MODULE)
+            skip = ('importlib*', '_structseq', TEST_MODULE)
             with TracerRun(self, skip=skip) as tracer:
                 tracer.runcall(tfunc_import)
 

From pypy.commits at gmail.com  Tue Oct 29 17:44:57 2019
From: pypy.commits at gmail.com (arigo)
Date: Tue, 29 Oct 2019 14:44:57 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: memory leak: c_data is assigned a
 str2charp() that was never freed
Message-ID: <5db8b2d9.1c69fb81.35d3b.790a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.6
Changeset: r97889:19641c3cf073
Date: 2019-10-29 22:44 +0100
http://bitbucket.org/pypy/pypy/changeset/19641c3cf073/

Log:	memory leak: c_data is assigned a str2charp() that was never freed

diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -101,13 +101,33 @@
 
 @slot_function([PyObject], lltype.Void)
 def unicode_dealloc(space, py_obj):
-    if get_wbuffer(py_obj):
+    if has_wbuffer_memory(py_obj):
         lltype.free(get_wbuffer(py_obj), flavor="raw")
-    if get_utf8(py_obj):
+    if has_utf8_memory(py_obj):
         lltype.free(get_utf8(py_obj), flavor="raw")
+    if not get_compact(py_obj) and get_data(py_obj):
+        lltype.free(get_data(py_obj), flavor="raw")
     from pypy.module.cpyext.object import _dealloc
     _dealloc(space, py_obj)
 
+def has_wbuffer_memory(py_obj):
+    ptr = get_wbuffer(py_obj)
+    if not ptr:
+        return False
+    elif not get_ready(py_obj):
+        return True
+    else:
+        return cts.cast('void *', ptr) != get_data(py_obj)
+
+def get_compact_ascii(py_obj):
+    return get_ascii(py_obj) and get_compact(py_obj)
+
+def has_utf8_memory(py_obj):
+    if get_compact_ascii(py_obj):
+        return False
+    utf8 = get_utf8(py_obj)
+    return bool(utf8) and cts.cast('void *', utf8) != get_data(py_obj)
+
 def get_len(py_obj):
     py_obj = cts.cast('PyASCIIObject*', py_obj)
     return py_obj.c_length
@@ -178,6 +198,12 @@
     py_obj = cts.cast('PyUnicodeObject*', py_obj)
     py_obj.c_data = p_data
 
+def get_compact(py_obj):
+    return rffi.getintfield(get_state(py_obj), 'c_compact')
+
+def set_compact(py_obj, value):
+    get_state(py_obj).c_compact = cts.cast('unsigned char', value)
+
 
 @cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL)
 def Py_UNICODE_ISSPACE(space, ch):
@@ -300,8 +326,8 @@
                     "Character U+%s is not in range [U+0000; U+10ffff]",
                     '%x' % maxchar)
     if maxchar < 256:
-        ucs1_data = rffi.str2charp(value)
-        set_data(py_obj, cts.cast('void*', ucs1_data))
+        ucs1_data = cts.cast('void *', rffi.str2charp(value))
+        set_data(py_obj, ucs1_data)
         set_kind(py_obj, _1BYTE_KIND)
         set_len(py_obj, get_wsize(py_obj))
         if maxchar < 128:
@@ -313,31 +339,33 @@
             set_utf8(py_obj, cts.cast('char *', 0))
             set_utf8_len(py_obj, 0)
     elif maxchar < 65536:
-        ucs2_str = utf8_encode_utf_16_helper(
-            value, 'strict',
-            byteorder=BYTEORDER)
-        if rffi.sizeof(lltype.UniChar) == 2 and not get_wbuffer(py_obj):
-            # Copy unicode buffer
-            wchar = cts.cast('wchar_t*', rffi.str2charp(ucs2_str))
-            set_wbuffer(py_obj, wchar)
-            set_wsize(py_obj, len(ucs2_str) // 2)
-        ucs2_data = cts.cast('Py_UCS2 *', rffi.str2charp(ucs2_str))
-        set_data(py_obj, cts.cast('void*', ucs2_data))
+        ucs2_data = cts.cast('void *', 0)
+        if rffi.sizeof(lltype.UniChar) == 2:
+            ucs2_data = cts.cast('void *', get_wbuffer(py_obj))
+        if not ucs2_data:
+            ucs2_str = utf8_encode_utf_16_helper(
+                value, 'strict',
+                byteorder=BYTEORDER)
+            ucs2_data = cts.cast('void *', rffi.str2charp(ucs2_str))
+            if rffi.sizeof(lltype.UniChar) == 2:
+                set_wbuffer(py_obj, cts.cast('wchar_t *', ucs2_data))
+        set_data(py_obj, ucs2_data)
         set_len(py_obj, get_wsize(py_obj))
         set_kind(py_obj, _2BYTE_KIND)
         set_utf8(py_obj, cts.cast('char *', 0))
         set_utf8_len(py_obj, 0)
     else:
-        ucs4_str = utf8_encode_utf_32_helper(
-            value, 'strict',
-            byteorder=BYTEORDER)
-        if rffi.sizeof(lltype.UniChar) == 4 and not get_wbuffer(py_obj):
-            # Copy unicode buffer
-            wchar = cts.cast('wchar_t*', rffi.str2charp(ucs4_str))
-            set_wbuffer(py_obj, wchar)
-            set_wsize(py_obj, len(ucs4_str) // 4)
-        ucs4_data = get_wbuffer(py_obj)
-        set_data(py_obj, cts.cast('void*', ucs4_data))
+        ucs4_data = cts.cast('void *', 0)
+        if rffi.sizeof(lltype.UniChar) == 4:
+            ucs4_data = cts.cast('void *', get_wbuffer(py_obj))
+        if not ucs4_data:
+            ucs4_str = utf8_encode_utf_32_helper(
+                value, 'strict',
+                byteorder=BYTEORDER)
+            ucs4_data = cts.cast('void *', rffi.str2charp(ucs4_str))
+            if rffi.sizeof(lltype.UniChar) == 4:
+                set_wbuffer(py_obj, cts.cast('wchar_t *', ucs4_data))
+        set_data(py_obj, ucs4_data)
         set_len(py_obj, get_wsize(py_obj))
         set_kind(py_obj, _4BYTE_KIND)
         set_utf8(py_obj, cts.cast('char *', 0))

From pypy.commits at gmail.com  Wed Oct 30 05:32:23 2019
From: pypy.commits at gmail.com (arigo)
Date: Wed, 30 Oct 2019 02:32:23 -0700 (PDT)
Subject: [pypy-commit] pypy default: Don't allocate a small list here
Message-ID: <5db958a7.1c69fb81.35105.6db6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r97890:195ceabd426e
Date: 2019-10-30 10:31 +0100
http://bitbucket.org/pypy/pypy/changeset/195ceabd426e/

Log:	Don't allocate a small list here

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -1184,9 +1184,9 @@
     size = len(s)
 
     if BYTEORDER == 'little':
-        iorder = [0, 1, 2, 3]
+        iorder0, iorder1, iorder2, iorder3 = 0, 1, 2, 3
     else:
-        iorder = [3, 2, 1, 0]
+        iorder0, iorder1, iorder2, iorder3 = 3, 2, 1, 0
 
     #  Check for BOM marks (U+FEFF) in the input and adjust current
     #  byte order setting accordingly. In native mode, the leading BOM
@@ -1196,8 +1196,8 @@
     if byteorder == 'native':
         if size >= 4:
             bom = intmask(
-                (ord(s[iorder[3]]) << 24) | (ord(s[iorder[2]]) << 16) |
-                (ord(s[iorder[1]]) << 8) | ord(s[iorder[0]]))
+                (ord(s[iorder3]) << 24) | (ord(s[iorder2]) << 16) |
+                (ord(s[iorder1]) << 8) | ord(s[iorder0]))
             if BYTEORDER == 'little':
                 if bom == BOM32_DIRECT:
                     pos += 4
@@ -1220,10 +1220,10 @@
         return '', 0, 0, bo
     if bo == -1:
         # force little endian
-        iorder = [0, 1, 2, 3]
+        iorder0, iorder1, iorder2, iorder3 = 0, 1, 2, 3
     elif bo == 1:
         # force big endian
-        iorder = [3, 2, 1, 0]
+        iorder0, iorder1, iorder2, iorder3 = 3, 2, 1, 0
 
     result = StringBuilder(size // 4)
 
@@ -1239,8 +1239,8 @@
             if len(s) - pos < 4:
                 break
             continue
-        ch = ((ord(s[pos + iorder[3]]) << 24) | (ord(s[pos + iorder[2]]) << 16) |
-              (ord(s[pos + iorder[1]]) << 8)  | ord(s[pos + iorder[0]]))
+        ch = ((ord(s[pos + iorder3]) << 24) | (ord(s[pos + iorder2]) << 16) |
+              (ord(s[pos + iorder1]) << 8)  | ord(s[pos + iorder0]))
         if not allow_surrogates and 0xD800 <= ch <= 0xDFFF:
             r, pos = errorhandler(errors, public_encoding_name,
                                   "code point in surrogate code point "

From pypy.commits at gmail.com  Wed Oct 30 05:33:23 2019
From: pypy.commits at gmail.com (arigo)
Date: Wed, 30 Oct 2019 02:33:23 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: PyUnicode_New()
Message-ID: <5db958e3.1c69fb81.1d2b4.79d4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.6
Changeset: r97891:699e6250c3cc
Date: 2019-10-30 10:32 +0100
http://bitbucket.org/pypy/pypy/changeset/699e6250c3cc/

Log:	PyUnicode_New()

diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -391,6 +391,38 @@
         s = module.asutf32(u)
         assert s == u.encode('utf-32')
 
+    def test_UnicodeNew(self):
+        module = self.import_extension('unicodenew', [
+            ("make", "METH_VARARGS",
+            """
+                long length = PyLong_AsLong(PyTuple_GetItem(args, 0));
+                long unichr = PyLong_AsLong(PyTuple_GetItem(args, 1));
+
+                PyObject *retval = PyUnicode_New(length, (Py_UCS4)unichr);
+                if (unichr <= 255) {
+                    Py_UCS1 *retbuf = PyUnicode_1BYTE_DATA(retval);
+                    for (long i = 0; i < length; i++)
+                        retbuf[i] = unichr;
+                }
+                else if (unichr <= 65535) {
+                    Py_UCS2 *retbuf = PyUnicode_2BYTE_DATA(retval);
+                    for (long i = 0; i < length; i++)
+                        retbuf[i] = unichr;
+                }
+                else {
+                    Py_UCS4 *retbuf = PyUnicode_4BYTE_DATA(retval);
+                    for (long i = 0; i < length; i++)
+                        retbuf[i] = unichr;
+                }
+                return retval;
+            """),
+            ])
+        assert module.make(0, 32) == u''
+        assert module.make(1, 32) == u' '
+        assert module.make(5, 255) == u'\xff' * 5
+        assert module.make(3, 0x1234) == u'\u1234' * 3
+        assert module.make(7, 0x12345) == u'\U00012345' * 7
+
 
 class TestUnicode(BaseApiTest):
     def test_unicodeobject(self, space):
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -8,12 +8,13 @@
 from pypy.interpreter.unicodehelper import (
     wcharpsize2utf8, str_decode_utf_16_helper, str_decode_utf_32_helper,
     unicode_encode_decimal, utf8_encode_utf_16_helper, BYTEORDER,
-    utf8_encode_utf_32_helper)
+    utf8_encode_utf_32_helper, str_decode_latin_1)
 from pypy.objspace.std.unicodeobject import unicodedb
 from pypy.module.cpyext.api import (
     CANNOT_FAIL, Py_ssize_t, build_type_checkers, cpython_api,
     bootstrap_function, CONST_STRING, INTP_real,
-    CONST_WSTRING, Py_CLEANUP_SUPPORTED, slot_function, cts, parse_dir)
+    CONST_WSTRING, Py_CLEANUP_SUPPORTED, slot_function, cts, parse_dir,
+    PyTypeObjectPtr)
 from pypy.module.cpyext.pyerrors import PyErr_BadArgument
 from pypy.module.cpyext.pyobject import (
     PyObject, PyObjectP, decref, make_ref, from_ref, track_reference,
@@ -22,6 +23,7 @@
 from pypy.module._codecs.interp_codecs import (
     CodecState, latin_1_decode, utf_16_decode, utf_32_decode)
 from pypy.objspace.std import unicodeobject
+from rpython.rlib.debug import fatalerror
 import sys
 
 ## See comment in bytesobject.py.
@@ -91,8 +93,31 @@
     Creates the unicode in the interpreter. The PyUnicodeObject buffer must not
     be modified after this call. Can raise in wcharpsize2utf8
     """
-    lgt = get_wsize(py_obj)
-    s_utf8 = wcharpsize2utf8(space, get_wbuffer(py_obj), lgt)
+    if not get_wbuffer(py_obj):
+        if not get_compact(py_obj):
+            fatalerror(
+                "internal cpyext error: realizing a non-compact unicode "
+                "object with wbuffer == null")
+        data = get_data(py_obj)
+        size = get_len(py_obj)
+        kind = get_kind(py_obj)
+        value = rffi.charpsize2str(data, size * kind)
+        if kind == _1BYTE_KIND:
+            s_utf8, lgt, _ = str_decode_latin_1(value, 'strict', True, None)
+        elif kind == _2BYTE_KIND:
+            decoded = str_decode_utf_16_helper(value, 'strict', True, None,
+                                               byteorder=BYTEORDER)
+            s_utf8, lgt = decoded[:2]
+        elif kind == _4BYTE_KIND:
+            decoded = str_decode_utf_32_helper(value, 'strict', True, None,
+                                               byteorder=BYTEORDER)
+            s_utf8, lgt = decoded[:2]
+        else:
+            assert False
+    else:
+        lgt = get_wsize(py_obj)
+        s_utf8 = wcharpsize2utf8(space, get_wbuffer(py_obj), lgt)
+
     w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type))
     w_obj = space.allocate_instance(unicodeobject.W_UnicodeObject, w_type)
     w_obj.__init__(s_utf8, lgt)
@@ -191,6 +216,15 @@
     py_obj.c_wstr_length = value
 
 def get_data(py_obj):
+    if get_compact(py_obj):
+        if get_ascii(py_obj):
+            PyASCIIObject = cts.gettype('PyASCIIObject')
+            struct_size = rffi.sizeof(PyASCIIObject)
+        else:
+            PyCompactUnicodeObject = cts.gettype('PyCompactUnicodeObject')
+            struct_size = rffi.sizeof(PyCompactUnicodeObject)
+        data = rffi.ptradd(rffi.cast(rffi.CCHARP, py_obj), struct_size)
+        return cts.cast('void *', data)
     py_obj = cts.cast('PyUnicodeObject*', py_obj)
     return py_obj.c_data
 
@@ -1138,3 +1172,64 @@
 def PyUnicode_AsUCS4Copy(space, ref):
     return PyUnicode_AsUCS4(space, ref, cts.cast('Py_UCS4*', 0), 0,
                             rffi.cast(rffi.INT_real, 1))
+
+ at cts.decl("PyObject* PyUnicode_New(Py_ssize_t size, Py_UCS4 maxchar)",
+          result_is_ll=True)
+def PyUnicode_New(space, size, maxchar):
+    PyASCIIObject = cts.gettype('PyASCIIObject')
+    PyCompactUnicodeObject = cts.gettype('PyCompactUnicodeObject')
+
+    is_ascii = False
+    is_sharing = False
+    struct_size = rffi.sizeof(PyCompactUnicodeObject)
+    if maxchar < 128:
+        kind = _1BYTE_KIND
+        char_size = 1
+        is_ascii = True
+        struct_size = rffi.sizeof(PyASCIIObject)
+    elif maxchar < 256:
+        kind = _1BYTE_KIND
+        char_size = 1
+    elif maxchar < 65536:
+        kind = _2BYTE_KIND
+        char_size = 2
+        if rffi.sizeof(lltype.UniChar) == 2:
+            is_sharing = True
+    else:
+        if maxchar > MAX_UNICODE:
+            raise oefmt(space.w_SystemError,
+                        "invalid maximum character passed to PyUnicode_New")
+        kind = _4BYTE_KIND
+        char_size = 4
+        if rffi.sizeof(lltype.UniChar) == 4:
+            is_sharing = True
+
+    # Ensure we won't overflow the size.
+    if size < 0:
+        raise oefmt(space.w_SystemError,
+                    "Negative size passed to PyUnicode_New")
+    if size > ((sys.maxint - struct_size) / char_size - 1):
+        raise oefmt(space.w_MemoryError, "PyUnicode_New: size too big")
+
+    # Duplicated allocation code from _PyObject_New() instead of a call to
+    # PyObject_New() so we are able to allocate space for the object and
+    # its data buffer.
+    pytype = as_pyobj(space, space.w_unicode)
+    pytype = rffi.cast(PyTypeObjectPtr, pytype)
+    buf = lltype.malloc(rffi.VOIDP.TO, struct_size + (size + 1) * char_size,
+                        flavor='raw', zero=True,
+                        add_memory_pressure=True)
+    pyobj = rffi.cast(PyObject, buf)
+    pyobj.c_ob_refcnt = 1
+    #pyobj.c_ob_pypy_link remains null for now
+    pyobj.c_ob_type = pytype
+
+    set_len(pyobj, size)
+    set_kind(pyobj, kind)
+    set_compact(pyobj, True)
+    set_ready(pyobj, True)
+    set_ascii(pyobj, is_ascii)
+    if is_sharing:
+        set_wsize(pyobj, size)
+        set_wbuffer(pyobj, rffi.cast(rffi.CWCHARP, get_data(pyobj)))
+    return pyobj

From pypy.commits at gmail.com  Wed Oct 30 12:53:51 2019
From: pypy.commits at gmail.com (mattip)
Date: Wed, 30 Oct 2019 09:53:51 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: issue 2970: add ncursesw to
 _minimal_curses
Message-ID: <5db9c01f.1c69fb81.1a4a8.3afe@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r97892:140497be04d3
Date: 2019-10-30 18:52 +0200
http://bitbucket.org/pypy/pypy/changeset/140497be04d3/

Log:	issue 2970: add ncursesw to _minimal_curses

diff --git a/lib_pypy/pyrepl/_minimal_curses.py b/lib_pypy/pyrepl/_minimal_curses.py
--- a/lib_pypy/pyrepl/_minimal_curses.py
+++ b/lib_pypy/pyrepl/_minimal_curses.py
@@ -16,7 +16,7 @@
 
 
 def _find_clib():
-    trylibs = ['ncurses', 'curses']
+    trylibs = ['ncursesw', 'ncurses', 'curses']
 
     for lib in trylibs:
         path = ctypes.util.find_library(lib)
diff --git a/pypy/module/_minimal_curses/fficurses.py b/pypy/module/_minimal_curses/fficurses.py
--- a/pypy/module/_minimal_curses/fficurses.py
+++ b/pypy/module/_minimal_curses/fficurses.py
@@ -14,6 +14,8 @@
     yield ExternalCompilationInfo(includes=['curses.h', 'term.h'])
     yield ExternalCompilationInfo(includes=['curses.h', 'term.h'],
                                   include_dirs=['/usr/include/ncurses'])
+    yield ExternalCompilationInfo(includes=['curses.h', 'term.h'],
+                                  include_dirs=['/usr/include/ncursesw'])
     yield ExternalCompilationInfo(includes=['ncurses/curses.h',
                                             'ncurses/term.h'])
 
@@ -23,6 +25,8 @@
     yield ExternalCompilationInfo(libraries=['ncurses'])
     yield ExternalCompilationInfo(libraries=['ncurses'],
                                   library_dirs=['/usr/lib64'])
+    yield ExternalCompilationInfo(libraries=['ncursesw'],
+                                  library_dirs=['/usr/lib64'])
 
 def try_tools():
     try:
@@ -30,6 +34,10 @@
     except Exception:
         pass
     try:
+        yield ExternalCompilationInfo.from_pkg_config("ncursesw")
+    except Exception:
+        pass
+    try:
         yield ExternalCompilationInfo.from_config_tool("ncurses5-config")
     except Exception:
         pass

From pypy.commits at gmail.com  Wed Oct 30 14:01:46 2019
From: pypy.commits at gmail.com (mattip)
Date: Wed, 30 Oct 2019 11:01:46 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: issue 2971: update from upstream package,
 should fix failing tests
Message-ID: <5db9d00a.1c69fb81.43501.53e9@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r97893:97cf5afa67c7
Date: 2019-10-30 20:00 +0200
http://bitbucket.org/pypy/pypy/changeset/97cf5afa67c7/

Log:	issue 2971: update from upstream package, should fix failing tests

diff --git a/extra_tests/test_pyrepl/infrastructure.py b/extra_tests/test_pyrepl/infrastructure.py
--- a/extra_tests/test_pyrepl/infrastructure.py
+++ b/extra_tests/test_pyrepl/infrastructure.py
@@ -59,6 +59,11 @@
         return Event(*ev)
 
 
+    def getpending(self):
+        """Nothing pending, but do not return None here."""
+        return Event('key', '', b'')
+
+
 class BaseTestReader(Reader):
 
     def get_prompt(self, lineno, cursor_on_line):
diff --git a/extra_tests/test_pyrepl/test_readline.py b/extra_tests/test_pyrepl/test_readline.py
--- a/extra_tests/test_pyrepl/test_readline.py
+++ b/extra_tests/test_pyrepl/test_readline.py
@@ -15,8 +15,7 @@
     os.write(master, b'input\n')
 
     with sane_term():
-        result = readline_wrapper.get_reader().readline()
-    #result = readline_wrapper.raw_input('prompt:')
+        result = readline_wrapper.raw_input('prompt:')
     assert result == 'input'
     # A bytes string on python2, a unicode string on python3.
     assert isinstance(result, str)
diff --git a/extra_tests/test_pyrepl/test_wishes.py b/extra_tests/test_pyrepl/test_wishes.py
--- a/extra_tests/test_pyrepl/test_wishes.py
+++ b/extra_tests/test_pyrepl/test_wishes.py
@@ -27,5 +27,5 @@
     read_spec([
         (('digit-arg', '3'),      ['']),
         (('quoted-insert', None), ['']),
-        (('self-insert', '\033'), ['^[^[^[']),
+        (('key', '\033'), ['^[^[^[']),
         (('accept', None),        None)])
diff --git a/lib_pypy/pyrepl/commands.py b/lib_pypy/pyrepl/commands.py
--- a/lib_pypy/pyrepl/commands.py
+++ b/lib_pypy/pyrepl/commands.py
@@ -369,8 +369,12 @@
 
 class qIHelp(Command):
     def do(self):
+        from .reader import disp_str
+
         r = self.reader
-        r.insert((self.event + r.console.getpending().data) * r.get_arg())
+        pending = r.console.getpending().data
+        disp = disp_str((self.event + pending).encode())[0]
+        r.insert(disp * r.get_arg())
         r.pop_input_trans()
 
 from pyrepl import input
diff --git a/lib_pypy/pyrepl/readline.py b/lib_pypy/pyrepl/readline.py
--- a/lib_pypy/pyrepl/readline.py
+++ b/lib_pypy/pyrepl/readline.py
@@ -259,7 +259,7 @@
         except _error:
             return _old_raw_input(prompt)
         reader.ps1 = prompt
-        return reader.readline(reader, startup_hook=self.startup_hook)
+        return reader.readline(returns_unicode=True, startup_hook=self.startup_hook)
 
     def multiline_input(self, more_lines, ps1, ps2, returns_unicode=False):
         """Read an input on possibly multiple lines, asking for more

From pypy.commits at gmail.com  Wed Oct 30 15:23:26 2019
From: pypy.commits at gmail.com (mattip)
Date: Wed, 30 Oct 2019 12:23:26 -0700 (PDT)
Subject: [pypy-commit] pypy default: issue 2970: add ncursesw to
 _minimal_curses
Message-ID: <5db9e32e.1c69fb81.182ad.71fa@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r97894:bba4a466169d
Date: 2019-10-30 21:21 +0200
http://bitbucket.org/pypy/pypy/changeset/bba4a466169d/

Log:	issue 2970: add ncursesw to _minimal_curses

diff --git a/pypy/module/_minimal_curses/fficurses.py b/pypy/module/_minimal_curses/fficurses.py
--- a/pypy/module/_minimal_curses/fficurses.py
+++ b/pypy/module/_minimal_curses/fficurses.py
@@ -14,6 +14,8 @@
     yield ExternalCompilationInfo(includes=['curses.h', 'term.h'])
     yield ExternalCompilationInfo(includes=['curses.h', 'term.h'],
                                   include_dirs=['/usr/include/ncurses'])
+    yield ExternalCompilationInfo(includes=['curses.h', 'term.h'],
+                                  include_dirs=['/usr/include/ncursesw'])
     yield ExternalCompilationInfo(includes=['ncurses/curses.h',
                                             'ncurses/term.h'])
 
@@ -23,6 +25,8 @@
     yield ExternalCompilationInfo(libraries=['ncurses'])
     yield ExternalCompilationInfo(libraries=['ncurses'],
                                   library_dirs=['/usr/lib64'])
+    yield ExternalCompilationInfo(libraries=['ncursesw'],
+                                  library_dirs=['/usr/lib64'])
 
 def try_tools():
     try:
@@ -30,6 +34,10 @@
     except Exception:
         pass
     try:
+        yield ExternalCompilationInfo.from_pkg_config("ncursesw")
+    except Exception:
+        pass
+    try:
         yield ExternalCompilationInfo.from_config_tool("ncurses5-config")
     except Exception:
         pass

From pypy.commits at gmail.com  Wed Oct 30 17:16:13 2019
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 30 Oct 2019 14:16:13 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix uses of raises()
Message-ID: <5db9fd9d.1c69fb81.ad90.9b6d@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r97897:ef97a01d2e01
Date: 2019-10-30 21:15 +0000
http://bitbucket.org/pypy/pypy/changeset/ef97a01d2e01/

Log:	Fix uses of raises()

diff --git a/pypy/interpreter/test/test_function.py b/pypy/interpreter/test/test_function.py
--- a/pypy/interpreter/test/test_function.py
+++ b/pypy/interpreter/test/test_function.py
@@ -76,7 +76,8 @@
                 return x
             return f
         f = g(42)
-        raises(TypeError, FuncType, f.func_code, f.func_globals, 'f2', None, None)
+        with raises(TypeError):
+            FuncType(f.func_code, f.func_globals, 'f2', None, None)
 
     def test_write_code(self):
         def f():
@@ -134,8 +135,10 @@
         assert res[1] == 22
         assert res[2] == 333
 
-        raises(TypeError, func)
-        raises(TypeError, func, 1, 2, 3, 4)
+        with raises(TypeError):
+            func()
+        with raises(TypeError):
+            func(1, 2, 3, 4)
 
     def test_simple_varargs(self):
         def func(arg1, *args):
@@ -162,7 +165,8 @@
     def test_kwargs_sets_wrong_positional_raises(self):
         def func(arg1):
             pass
-        raises(TypeError, func, arg2=23)
+        with raises(TypeError):
+            func(arg2=23)
 
     def test_kwargs_sets_positional(self):
         def func(arg1):
@@ -180,8 +184,8 @@
     def test_kwargs_sets_positional_twice(self):
         def func(arg1, **kw):
             return arg1, kw
-        raises(
-            TypeError, func, 42, {'arg1': 23})
+        with raises(TypeError):
+            func(42, {'arg1': 23})
 
     def test_kwargs_nondict_mapping(self):
         class Mapping:
@@ -194,9 +198,10 @@
         res = func(23, **Mapping())
         assert res[0] == 23
         assert res[1] == {'a': 'a', 'b': 'b'}
-        error = raises(TypeError, lambda: func(42, **[]))
-        assert error.value.message == ('argument after ** must be a mapping, '
-                                       'not list')
+        with raises(TypeError) as excinfo:
+            func(42, **[])
+        assert excinfo.value.message == (
+            'argument after ** must be a mapping, not list')
 
     def test_default_arg(self):
         def func(arg1,arg2=42):
@@ -215,12 +220,14 @@
     def test_defaults_keyword_override_but_leaves_empty_positional(self):
         def func(arg1,arg2=42):
             return arg1, arg2
-        raises(TypeError, func, arg2=23)
+        with raises(TypeError):
+            func(arg2=23)
 
     def test_kwargs_disallows_same_name_twice(self):
         def func(arg1, **kw):
             return arg1, kw
-        raises(TypeError, func, 42, **{'arg1': 23})
+        with raises(TypeError):
+            func(42, **{'arg1': 23})
 
     def test_kwargs_bound_blind(self):
         class A(object):
@@ -269,15 +276,21 @@
 
     def test_call_builtin(self):
         s = 'hello'
-        raises(TypeError, len)
+        with raises(TypeError):
+            len()
         assert len(s) == 5
-        raises(TypeError, len, s, s)
-        raises(TypeError, len, s, s, s)
+        with raises(TypeError):
+            len(s, s)
+        with raises(TypeError):
+            len(s, s, s)
         assert len(*[s]) == 5
         assert len(s, *[]) == 5
-        raises(TypeError, len, some_unknown_keyword=s)
-        raises(TypeError, len, s, some_unknown_keyword=s)
-        raises(TypeError, len, s, s, some_unknown_keyword=s)
+        with raises(TypeError):
+            len(some_unknown_keyword=s)
+        with raises(TypeError):
+            len(s, some_unknown_keyword=s)
+        with raises(TypeError):
+            len(s, s, some_unknown_keyword=s)
 
     def test_call_error_message(self):
         try:
@@ -310,8 +323,10 @@
         # cannot subclass 'function' or 'builtin_function'
         def f():
             pass
-        raises(TypeError, type, 'Foo', (type(f),), {})
-        raises(TypeError, type, 'Foo', (type(len),), {})
+        with raises(TypeError):
+            type('Foo', (type(f),), {})
+        with raises(TypeError):
+            type('Foo', (type(len),), {})
 
     def test_lambda_docstring(self):
         # Like CPython, (lambda:"foo") has a docstring of "foo".
@@ -324,7 +339,8 @@
         f = lambda: 42
         # not sure what it should raise, since CPython doesn't have setstate
         # on function types
-        raises(ValueError, type(f).__setstate__, f, (1, 2, 3))
+        with raises(ValueError):
+            type(f).__setstate__(f, (1, 2, 3))
 
 class AppTestMethod:
     def test_simple_call(self):
@@ -471,14 +487,13 @@
 
         assert A.foo(A(), 42) == (42,)
         assert A.foo(B(), 42) == (42,)
-        raises(TypeError, A.foo, 5)
-        raises(TypeError, B.foo, C())
-        try:
+        with raises(TypeError):
+            A.foo(5)
+        with raises(TypeError):
+            B.foo(C())
+        with raises(TypeError):
             class Fun:
                 __metaclass__ = A.foo
-            assert 0  # should have raised
-        except TypeError:
-            pass
         class Fun:
             __metaclass__ = A().foo
         assert Fun[:2] == ('Fun', ())
@@ -488,14 +503,18 @@
         def f(*args):
             return args
         m = new.instancemethod(f, None, "foobar")
-        raises(TypeError, m)
-        raises(TypeError, m, None)
-        raises(TypeError, m, "egg")
+        with raises(TypeError):
+            m()
+        with raises(TypeError):
+            m(None)
+        with raises(TypeError):
+            m("egg")
 
         m = new.instancemethod(f, None, (str, int))     # really obscure...
         assert m(4) == (4,)
         assert m("uh") == ("uh",)
-        raises(TypeError, m, [])
+        with raises(TypeError):
+            m([])
 
         class MyBaseInst(object):
             pass
@@ -518,22 +537,30 @@
         x = MyInst(BSub1)
         m = new.instancemethod(f, None, BSub1)
         assert m(x) == (x,)
-        raises(TypeError, m, MyInst(BBase))
-        raises(TypeError, m, MyInst(BSub2))
-        raises(TypeError, m, MyInst(None))
-        raises(TypeError, m, MyInst(42))
+        with raises(TypeError):
+            m(MyInst(BBase))
+        with raises(TypeError):
+            m(MyInst(BSub2))
+        with raises(TypeError):
+            m(MyInst(None))
+        with raises(TypeError):
+            m(MyInst(42))
 
     def test_invalid_creation(self):
         import new
-        def f(): pass
-        raises(TypeError, new.instancemethod, f, None)
+        def f():
+            pass
+        with raises(TypeError):
+            new.instancemethod(f, None)
 
     def test_empty_arg_kwarg_call(self):
         def f():
             pass
 
-        raises(TypeError, lambda: f(*0))
-        raises(TypeError, lambda: f(**0))
+        with raises(TypeError):
+            f(*0)
+        with raises(TypeError):
+            f(**0)
 
     def test_method_equal(self):
         class A(object):

From pypy.commits at gmail.com  Thu Oct 31 01:20:24 2019
From: pypy.commits at gmail.com (mattip)
Date: Wed, 30 Oct 2019 22:20:24 -0700 (PDT)
Subject: [pypy-commit] buildbot default: add a dockerfile to run a
 centos6-based x64 buildslave
Message-ID: <5dba6f18.1c69fb81.90057.117d@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r1097:709f8fb564ac
Date: 2019-10-31 07:12 +0200
http://bitbucket.org/pypy/buildbot/changeset/709f8fb564ac/

Log:	add a dockerfile to run a centos6-based x64 buildslave

diff --git a/README_BUILDSLAVE b/README_BUILDSLAVE
--- a/README_BUILDSLAVE
+++ b/README_BUILDSLAVE
@@ -1,7 +1,13 @@
 How to setup a buildslave for PyPy
 ==================================
 
-The required setup is::
+There are three documented ways to run a PyPy buildslave:
+
+- "bare metal" on the host machine, described below
+- in a chroot, setting up the chroot is described in `README-CHROOT`
+- via docker on a centos6-based image, see docker/Dockerfile
+
+In all cases the required setup is::
 - hg
 - "python" should run a cpython 2.6-2.7 that will run the test suites.
 - virtualenv that will use the "python" from above
diff --git a/docker/Dockerfile b/docker/Dockerfile
new file mode 100644
--- /dev/null
+++ b/docker/Dockerfile
@@ -0,0 +1,46 @@
+# build with something like this, where
+#  -t is the name of the image
+#  -f is this file
+#  the . is a random directory
+# docker build -t buildslave -f docker/Dockerfile .
+#
+# To create the buildslave configuration, call
+# docker run --rm -v<build_dir>:/build_dir \
+#    -eSLAVENAME=<name> -ePASSWORD=<password> buildslave
+# Then you can examine the <build_dir>/buildbot.tac file.
+#
+# To run the buildslave (after the stage above succeeds) you no longer need the
+# SLAVENAME and PASSWORD. The slave will run non-deamonized, which will appear
+# to "hang" the console running the slave. To stop the slave, simply CTRL-C or
+# kill the process.
+#
+# To enter the buildslave image
+# docker run -it -v<abspath/to/builder/dir>:/build_dir> buildslave /bin/bash
+
+FROM centos:centos6
+
+RUN yum -y update
+RUN yum install -y wget bzip2-devel zlib-devel glibc-devel libX11-devel \
+   libXt-devel patch expat-devel libXft-devel openssl-devel tk-devel gdbm-devel \
+   perl xz-devel libffi-devel ncurses-devel sqlite-devel
+RUN yum install -y centos-release-scl
+
+RUN wget https://github.com/squeaky-pl/centos-devtools/releases/download/8.2-s1/gcc-8.2.0-binutils-2.32-x86_64.tar.bz2 -O - | tar -C / -xj
+RUN wget https://bitbucket.org/squeaky/portable-pypy/downloads/pypy-7.0.0-linux_x86_64-portable.tar.bz2 -O - | tar -C /opt -xj
+RUN ln -s /opt/pypy-7.0.0-linux_x86_64-portable/bin/pypy /usr/local/bin/pypy
+
+RUN yum install -y python27 python27-python-virtualenv
+
+#yuck
+ENV LD_LIBRARY_PATH=/opt/rh/python27/root/usr/lib64
+RUN /opt/rh/python27/root/usr/bin/python -mvirtualenv /python27_virt
+ENV PATH=/python27_virt/bin:$PATH
+ENV PATH=/opt/devtools-8.2/bin:$PATH
+RUN pip install --upgrade pip setuptools
+RUN pip install buildbot-slave pytest hypothesis cffi vmprof mercurial
+
+CMD if [ -e /build_dir/buildbot.tac ]; then \
+    buildslave start --nodaemon /build_dir; \
+  else \
+    buildslave create-slave /build_dir buildbot.pypy.org:10407 $SLAVENAME $PASSWORD; \
+  fi

From pypy.commits at gmail.com  Thu Oct 31 07:23:51 2019
From: pypy.commits at gmail.com (mattip)
Date: Thu, 31 Oct 2019 04:23:51 -0700 (PDT)
Subject: [pypy-commit] pypy default: backport ncurses changes from
 portable-pypy
Message-ID: <5dbac447.1c69fb81.11585.60f3@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r97905:28382af84334
Date: 2019-10-31 13:00 +0200
http://bitbucket.org/pypy/pypy/changeset/28382af84334/

Log:	backport ncurses changes from portable-pypy

diff --git a/lib_pypy/_curses_build.py b/lib_pypy/_curses_build.py
--- a/lib_pypy/_curses_build.py
+++ b/lib_pypy/_curses_build.py
@@ -1,14 +1,28 @@
 from cffi import FFI
 import os
 
-# On some systems, the ncurses library is
-# located at /usr/include/ncurses, so we must check this case.
-# Let's iterate over well known paths
-incdirs =  []
-for _path in ['/usr/include', '/usr/include/ncurses']:
-    if os.path.isfile(os.path.join(_path, 'panel.h')):
-        incdirs.append(_path)
-        break
+def find_library(options):
+    for library in options:
+        ffi = FFI()
+        ffi.set_source("_curses_cffi_check", "", libraries=[library])
+        try:
+            ffi.compile()
+        except VerificationError as e:
+            e_last = e
+            continue
+        else:
+            return library
+
+    # If none of the options is available, present the user a meaningful
+    # error message
+    raise e_last
+
+def find_curses_include_dirs():
+    if os.path.exists('/usr/include/ncurses'):
+        return ['/usr/include/ncurses']
+    if os.path.exists('/usr/include/ncursesw'):
+    	return ['/usr/include/ncursesw']
+    return []
 
 
 ffi = FFI()
@@ -59,8 +73,9 @@
 void _m_getsyx(int *yx) {
     getsyx(yx[0], yx[1]);
 }
-""", include_dirs=incdirs, 
-     libraries=['ncurses', 'panel'])
+""", libraries=[find_library(['ncurses', 'ncursesw']),
+                find_library(['panel', 'panelw'])],
+     include_dirs=find_curses_include_dirs())
 
 
 ffi.cdef("""
@@ -71,6 +86,8 @@
 typedef unsigned long... chtype;
 typedef chtype attr_t;
 
+typedef int... wint_t;
+
 typedef struct
 {
     short id;           /* ID to distinguish multiple devices */
@@ -82,6 +99,7 @@
 static const int ERR, OK;
 static const int TRUE, FALSE;
 static const int KEY_MIN, KEY_MAX;
+static const int KEY_CODE_YES;
 
 static const int COLOR_BLACK;
 static const int COLOR_RED;
@@ -168,6 +186,8 @@
 void filter(void);
 int flash(void);
 int flushinp(void);
+int wget_wch(WINDOW *, wint_t *);
+int mvwget_wch(WINDOW *, int, int, wint_t *);
 chtype getbkgd(WINDOW *);
 WINDOW * getwin(FILE *);
 int halfdelay(int);
@@ -243,6 +263,7 @@
 int touchwin(WINDOW *);
 int typeahead(int);
 int ungetch(int);
+int unget_wch(const wchar_t);
 int untouchwin(WINDOW *);
 void use_env(bool);
 int waddch(WINDOW *, const chtype);

From pypy.commits at gmail.com  Thu Oct 31 07:47:10 2019
From: pypy.commits at gmail.com (mattip)
Date: Thu, 31 Oct 2019 04:47:10 -0700 (PDT)
Subject: [pypy-commit] buildbot default: update docker run for --user
Message-ID: <5dbac9be.1c69fb81.a2525.7715@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r1098:968866aaeff1
Date: 2019-10-31 13:46 +0200
http://bitbucket.org/pypy/buildbot/changeset/968866aaeff1/

Log:	update docker run for --user

diff --git a/docker/Dockerfile b/docker/Dockerfile
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -5,7 +5,7 @@
 # docker build -t buildslave -f docker/Dockerfile .
 #
 # To create the buildslave configuration, call
-# docker run --rm -v<build_dir>:/build_dir \
+# docker run --rm --user $UID -v<abspath/to/builder/dir>:/build_dir> \
 #    -eSLAVENAME=<name> -ePASSWORD=<password> buildslave
 # Then you can examine the <build_dir>/buildbot.tac file.
 #
@@ -15,7 +15,8 @@
 # kill the process.
 #
 # To enter the buildslave image
-# docker run -it -v<abspath/to/builder/dir>:/build_dir> buildslave /bin/bash
+# docker run -it --user $UID -v<abspath/to/builder/dir>:/build_dir> \
+#     buildslave /bin/bash
 
 FROM centos:centos6
 

From pypy.commits at gmail.com  Thu Oct 31 09:14:26 2019
From: pypy.commits at gmail.com (mattip)
Date: Thu, 31 Oct 2019 06:14:26 -0700 (PDT)
Subject: [pypy-commit] pypy _PySet_NextEntry: start to implement
 _PySet_NextEntry
Message-ID: <5dbade32.1c69fb81.1b2c0.992d@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: _PySet_NextEntry
Changeset: r97906:386490448614
Date: 2019-10-31 15:13 +0200
http://bitbucket.org/pypy/pypy/changeset/386490448614/

Log:	start to implement _PySet_NextEntry

diff --git a/pypy/module/cpyext/setobject.py b/pypy/module/cpyext/setobject.py
--- a/pypy/module/cpyext/setobject.py
+++ b/pypy/module/cpyext/setobject.py
@@ -7,6 +7,34 @@
 from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall
 from pypy.objspace.std.setobject import W_SetObject, W_FrozensetObject, newset
 
+PySetObjectStruct = lltype.ForwardReference()
+PySetObject = lltype.Ptr(PySetObjectStruct)
+PySetObjectFields = PyObjectFields + \
+    (("_tmpset", PyObject),)
+cpython_struct("PySetObject", PySetObjectFields, PySetObjectStruct)
+
+ at bootstrap_function
+def init_dictobject(space):
+    "Type description of PyDictObject"
+    make_typedescr(space.w_dict.layout.typedef,
+                   basestruct=PySetObject.TO,
+                   attach=set_attach,
+                   dealloc=set_dealloc,
+                  )
+
+def set_attach(space, py_obj, w_obj, w_userdata=None):
+    """
+    Fills a newly allocated PySetObject with the given set object.
+    """
+    py_set = rffi.cast(PySettObject, py_obj)
+    py_set.c__tmpset = lltype.nullptr(PyObject.TO)
+
+ at slot_function([PyObject], lltype.Void)
+def dict_dealloc(space, py_obj):
+    py_set = rffi.cast(PySetObject, py_obj)
+    decref(space, py_set.c__tmpset)
+    py_set.c__tmpset = lltype.nullptr(PyObject.TO)
+    _dealloc(space, py_obj)
 
 PySet_Check, PySet_CheckExact = build_type_checkers("Set")
 PyFrozenSet_Check, PyFrozenSet_CheckExact = build_type_checkers("FrozenSet")
@@ -23,7 +51,7 @@
     """Return true if obj is a set object or a frozenset object but
     not an instance of a subtype."""
     w_obj_type = space.type(w_obj)
-    return (space.is_w(w_obj_type, space.gettypefor(W_SetObject)) or 
+    return (space.is_w(w_obj_type, space.gettypefor(W_SetObject)) or
             space.is_w(w_obj_type, space.gettypefor(W_FrozensetObject)))
 
 @cpython_api([PyObject], PyObject)
@@ -126,4 +154,30 @@
     else:
         return space.call_function(space.w_frozenset, w_iterable)
 
-
+ at cpython_api([PyObject, Py_ssize_tP, PyObjectP, Py_ssize_tP], rffi.INT_real, error=-1)
+def _PySet_NextEntry(space, w_set, ppos, pkey, phash):
+    if w_set is None or not space.isinstance_w(w_set, space.w_set):
+        PyErr_BadInternalCall(space)
+        return -1
+    if not key:
+        PyErr_BadInternalCall(space)
+        return -1
+    py_obj = as_pyobj(space, w_set)
+    py_set = rffi.cast(PySetObject, py_obj)
+    if not py_set.c__tmpkeys:
+        w_keyview = space.call_method(space.w_set, "list", w_set)
+        # w_keys must use the object strategy in order to keep the keys alive
+        w_keys = space.newlist(space.listview(w_keyview))
+        w_keys.switch_to_object_strategy()
+        py_set.c__tmpkeys = create_ref(space, w_keys)
+        incref(space, py_set.c__tmpkeys)
+    else:
+        w_keys = from_ref(space, py_set.c__tmpkeys)
+    pos = pos[0]
+    if pos >= space.len_w(w_keys):
+        return 0
+    w_key = space.listview(w_keys)[pos]
+    key[0] = create_ref(space, w_key)
+    # steals a reference
+    hash[0] = -1 # meaningless in PyPy
+    return 1

From pypy.commits at gmail.com  Thu Oct 31 09:25:06 2019
From: pypy.commits at gmail.com (arigo)
Date: Thu, 31 Oct 2019 06:25:06 -0700 (PDT)
Subject: [pypy-commit] pypy _PySet_NextEntry: in-progress
Message-ID: <5dbae0b2.1c69fb81.2fa61.bce2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: _PySet_NextEntry
Changeset: r97907:4b97719a1d45
Date: 2019-10-31 14:24 +0100
http://bitbucket.org/pypy/pypy/changeset/4b97719a1d45/

Log:	in-progress

diff --git a/pypy/module/cpyext/include/Python.h b/pypy/module/cpyext/include/Python.h
--- a/pypy/module/cpyext/include/Python.h
+++ b/pypy/module/cpyext/include/Python.h
@@ -108,6 +108,7 @@
 #include "descrobject.h"
 #include "tupleobject.h"
 #include "dictobject.h"
+#include "setobject.h"
 #include "longobject.h"
 #include "listobject.h"
 #include "longobject.h"
diff --git a/pypy/module/cpyext/setobject.py b/pypy/module/cpyext/setobject.py
--- a/pypy/module/cpyext/setobject.py
+++ b/pypy/module/cpyext/setobject.py
@@ -1,39 +1,43 @@
 from pypy.interpreter.error import OperationError, oefmt
 from rpython.rtyper.lltypesystem import rffi, lltype
-from pypy.module.cpyext.api import (cpython_api, Py_ssize_t, CANNOT_FAIL,
-                                    build_type_checkers)
+from pypy.module.cpyext.api import (
+    cpython_api, Py_ssize_t, Py_ssize_tP, CANNOT_FAIL, build_type_checkers,
+    PyObjectFields, cpython_struct, bootstrap_function, slot_function)
 from pypy.module.cpyext.pyobject import (PyObject, PyObjectP,
-    make_ref, from_ref)
+    make_ref, from_ref, make_typedescr)
 from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall
 from pypy.objspace.std.setobject import W_SetObject, W_FrozensetObject, newset
 
 PySetObjectStruct = lltype.ForwardReference()
 PySetObject = lltype.Ptr(PySetObjectStruct)
 PySetObjectFields = PyObjectFields + \
-    (("_tmpset", PyObject),)
+    (("_tmplist", PyObject),)
 cpython_struct("PySetObject", PySetObjectFields, PySetObjectStruct)
 
 @bootstrap_function
 def init_dictobject(space):
-    "Type description of PyDictObject"
-    make_typedescr(space.w_dict.layout.typedef,
+    "Type description of PySetObject"
+    make_typedescr(space.w_set.layout.typedef,
                    basestruct=PySetObject.TO,
                    attach=set_attach,
-                   dealloc=set_dealloc,
-                  )
+                   dealloc=set_dealloc)
+    make_typedescr(space.w_frozenset.layout.typedef,   # same as 'set'
+                   basestruct=PySetObject.TO,
+                   attach=set_attach,
+                   dealloc=set_dealloc)
 
 def set_attach(space, py_obj, w_obj, w_userdata=None):
     """
     Fills a newly allocated PySetObject with the given set object.
     """
-    py_set = rffi.cast(PySettObject, py_obj)
-    py_set.c__tmpset = lltype.nullptr(PyObject.TO)
+    py_set = rffi.cast(PySetObject, py_obj)
+    py_set.c__tmplist = lltype.nullptr(PyObject.TO)
 
 @slot_function([PyObject], lltype.Void)
-def dict_dealloc(space, py_obj):
+def set_dealloc(space, py_obj):
     py_set = rffi.cast(PySetObject, py_obj)
-    decref(space, py_set.c__tmpset)
-    py_set.c__tmpset = lltype.nullptr(PyObject.TO)
+    decref(space, py_set.c__tmplist)
+    py_set.c__tmplist = lltype.nullptr(PyObject.TO)
     _dealloc(space, py_obj)
 
 PySet_Check, PySet_CheckExact = build_type_checkers("Set")
@@ -164,15 +168,15 @@
         return -1
     py_obj = as_pyobj(space, w_set)
     py_set = rffi.cast(PySetObject, py_obj)
-    if not py_set.c__tmpkeys:
+    if not py_set.c__tmplist:
         w_keyview = space.call_method(space.w_set, "list", w_set)
         # w_keys must use the object strategy in order to keep the keys alive
         w_keys = space.newlist(space.listview(w_keyview))
         w_keys.switch_to_object_strategy()
-        py_set.c__tmpkeys = create_ref(space, w_keys)
-        incref(space, py_set.c__tmpkeys)
+        py_set.c__tmplist = create_ref(space, w_keys)
+        incref(space, py_set.c__tmplist)
     else:
-        w_keys = from_ref(space, py_set.c__tmpkeys)
+        w_keys = from_ref(space, py_set.c__tmplist)
     pos = pos[0]
     if pos >= space.len_w(w_keys):
         return 0
diff --git a/pypy/module/cpyext/test/test_setobject.py b/pypy/module/cpyext/test/test_setobject.py
--- a/pypy/module/cpyext/test/test_setobject.py
+++ b/pypy/module/cpyext/test/test_setobject.py
@@ -3,6 +3,7 @@
 from pypy.module.cpyext.setobject import (
     PySet_Check, PyFrozenSet_Check, PyFrozenSet_CheckExact,
     PySet_Add, PySet_Size, PySet_GET_SIZE)
+from pypy.module.cpyext.api import Py_ssize_tP, PyObjectP
 
 
 class TestTupleObject(BaseApiTest):
@@ -62,6 +63,22 @@
         """)
         assert api.PyAnySet_Check(w_instance)
 
+    def test_pyset_next(self, space, api):
+        w_set = space.call_function(space.w_set, space.newtext("ab"))
+        with rffi.scoped_alloc(Py_ssize_tP.TO, 1) as pos_p:
+            with rffi.scoped_alloc(PyObjectP.TO, 1) as result_p:
+                pos_p[0] = 0
+                res = api._PySet_Next(w_set, pos_p, result_p)
+                assert res == 1
+                letter1 = space.text_w(result_p[0])
+                res = api._PySet_Next(w_set, pos_p, result_p)
+                assert res == 1
+                letter2 = space.text_w(result_p[0])
+                res = api._PySet_Next(w_set, pos_p, result_p)
+                assert res == 0
+        assert set([letter1, letter2]) == set("ab")
+
+
 class AppTestSetObject(AppTestCpythonExtensionBase):
     def test_set_macro_cast(self):
         module = self.import_extension('foo', [

From pypy.commits at gmail.com  Thu Oct 31 12:13:32 2019
From: pypy.commits at gmail.com (arigo)
Date: Thu, 31 Oct 2019 09:13:32 -0700 (PDT)
Subject: [pypy-commit] pypy default: Add _PySet_Next{,
 Entry}.  Based on the _PySet_NextEntry branch but for pypy2
Message-ID: <5dbb082c.1c69fb81.ebff.c265@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r97908:4748c02be810
Date: 2019-10-31 17:09 +0100
http://bitbucket.org/pypy/pypy/changeset/4748c02be810/

Log:	Add _PySet_Next{,Entry}. Based on the _PySet_NextEntry branch but
	for pypy2

diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -267,14 +267,15 @@
     if pos == 0:
         # Store the current keys in the PyDictObject.
         from pypy.objspace.std.listobject import W_ListObject
-        decref(space, py_dict.c__tmpkeys)
         w_keys = space.call_method(space.w_dict, "keys", w_dict)
         # w_keys must use the object strategy in order to keep the keys alive
         if not isinstance(w_keys, W_ListObject):
             return 0     # XXX should not call keys() above
         w_keys.switch_to_object_strategy()
+        oldkeys = py_dict.c__tmpkeys
         py_dict.c__tmpkeys = create_ref(space, w_keys)
         incref(space, py_dict.c__tmpkeys)
+        decref(space, oldkeys)
     else:
         if not py_dict.c__tmpkeys:
             # pos should have been 0, cannot fail so return 0
diff --git a/pypy/module/cpyext/include/Python.h b/pypy/module/cpyext/include/Python.h
--- a/pypy/module/cpyext/include/Python.h
+++ b/pypy/module/cpyext/include/Python.h
@@ -107,6 +107,7 @@
 #include "descrobject.h"
 #include "tupleobject.h"
 #include "dictobject.h"
+#include "setobject.h"
 #include "intobject.h"
 #include "listobject.h"
 #include "longobject.h"
diff --git a/pypy/module/cpyext/include/setobject.h b/pypy/module/cpyext/include/setobject.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/include/setobject.h
@@ -0,0 +1,19 @@
+
+/* set object interface */
+
+#ifndef Py_SETOBJECT_H
+#define Py_SETOBJECT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+    PyObject_HEAD
+    PyObject *_tmplist; /* a private place to put values during _PySet_Next */
+} PySetObject;
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_SETOBJECT_H */
+
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -359,11 +359,11 @@
     """
     Compute and return the hash value of an object o.  On failure, return -1.
     This is the equivalent of the Python expression hash(o)."""
-    return space.int_w(space.hash(w_obj))
+    return space.hash_w(w_obj)
 
 @cpython_api([rffi.DOUBLE], rffi.LONG, error=-1)
 def _Py_HashDouble(space, v):
-    return space.int_w(space.hash(space.newfloat(v)))
+    return space.hash_w(space.newfloat(v))
 
 @cpython_api([PyObject], lltype.Signed, error=-1)
 def PyObject_HashNotImplemented(space, o):
diff --git a/pypy/module/cpyext/setobject.py b/pypy/module/cpyext/setobject.py
--- a/pypy/module/cpyext/setobject.py
+++ b/pypy/module/cpyext/setobject.py
@@ -1,12 +1,45 @@
 from pypy.interpreter.error import OperationError, oefmt
 from rpython.rtyper.lltypesystem import rffi, lltype
-from pypy.module.cpyext.api import (cpython_api, Py_ssize_t, CANNOT_FAIL,
-                                    build_type_checkers)
+from pypy.module.cpyext.api import (
+    cpython_api, Py_ssize_t, Py_ssize_tP, CANNOT_FAIL, build_type_checkers,
+    PyObjectFields, cpython_struct, bootstrap_function, slot_function)
 from pypy.module.cpyext.pyobject import (PyObject, PyObjectP,
-    make_ref, from_ref)
+    make_ref, from_ref, as_pyobj, create_ref, make_typedescr, incref, decref)
+from pypy.module.cpyext.object import _dealloc
 from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall
 from pypy.objspace.std.setobject import W_SetObject, W_FrozensetObject, newset
 
+PySetObjectStruct = lltype.ForwardReference()
+PySetObject = lltype.Ptr(PySetObjectStruct)
+PySetObjectFields = PyObjectFields + \
+    (("_tmplist", PyObject),)
+cpython_struct("PySetObject", PySetObjectFields, PySetObjectStruct)
+
+ at bootstrap_function
+def init_setobject(space):
+    "Type description of PySetObject"
+    make_typedescr(space.w_set.layout.typedef,
+                   basestruct=PySetObject.TO,
+                   attach=set_attach,
+                   dealloc=set_dealloc)
+    make_typedescr(space.w_frozenset.layout.typedef,   # same as 'set'
+                   basestruct=PySetObject.TO,
+                   attach=set_attach,
+                   dealloc=set_dealloc)
+
+def set_attach(space, py_obj, w_obj, w_userdata=None):
+    """
+    Fills a newly allocated PySetObject with the given set object.
+    """
+    py_set = rffi.cast(PySetObject, py_obj)
+    py_set.c__tmplist = lltype.nullptr(PyObject.TO)
+
+ at slot_function([PyObject], lltype.Void)
+def set_dealloc(space, py_obj):
+    py_set = rffi.cast(PySetObject, py_obj)
+    decref(space, py_set.c__tmplist)
+    py_set.c__tmplist = lltype.nullptr(PyObject.TO)
+    _dealloc(space, py_obj)
 
 PySet_Check, PySet_CheckExact = build_type_checkers("Set")
 PyFrozenSet_Check, PyFrozenSet_CheckExact = build_type_checkers("FrozenSet")
@@ -23,7 +56,7 @@
     """Return true if obj is a set object or a frozenset object but
     not an instance of a subtype."""
     w_obj_type = space.type(w_obj)
-    return (space.is_w(w_obj_type, space.gettypefor(W_SetObject)) or 
+    return (space.is_w(w_obj_type, space.gettypefor(W_SetObject)) or
             space.is_w(w_obj_type, space.gettypefor(W_FrozensetObject)))
 
 @cpython_api([PyObject], PyObject)
@@ -126,4 +159,43 @@
     else:
         return space.call_function(space.w_frozenset, w_iterable)
 
+ at cpython_api([PyObject, Py_ssize_tP, PyObjectP, Py_ssize_tP], rffi.INT_real, error=-1)
+def _PySet_NextEntry(space, w_set, ppos, pkey, phash):
+    if w_set is None or not PyAnySet_Check(space, w_set):
+        PyErr_BadInternalCall(space)
+        return -1
+    if not pkey:
+        PyErr_BadInternalCall(space)
+        return -1
+    pos = ppos[0]
+    py_obj = as_pyobj(space, w_set)
+    py_set = rffi.cast(PySetObject, py_obj)
+    if pos == 0:
+        # Store the current item list in the PySetObject.
+        # w_keys must use the object strategy in order to keep the keys alive
+        w_keys = space.newlist(space.listview(w_set))
+        w_keys.switch_to_object_strategy()
+        oldlist = py_set.c__tmplist
+        py_set.c__tmplist = create_ref(space, w_keys)
+        incref(space, py_set.c__tmplist)
+        decref(space, oldlist)
+    else:
+        if not py_set.c__tmplist:
+            # pos should have been 0, cannot fail so return 0
+            return 0;
+        w_keys = from_ref(space, py_set.c__tmplist)
+    ppos[0] += 1
+    if pos >= space.len_w(w_keys):
+        decref(space, py_set.c__tmplist)
+        py_set.c__tmplist = lltype.nullptr(PyObject.TO)
+        return 0
+    w_key = space.listview(w_keys)[pos]
+    pkey[0] = as_pyobj(space, w_key)
+    if phash:
+        phash[0] = space.hash_w(w_key)
+    return 1
 
+ at cpython_api([PyObject, Py_ssize_tP, PyObjectP], rffi.INT_real, error=-1)
+def _PySet_Next(space, w_set, ppos, pkey):
+    null = lltype.nullptr(Py_ssize_tP.TO)
+    return _PySet_NextEntry(space, w_set, ppos, pkey, null)
diff --git a/pypy/module/cpyext/test/test_setobject.py b/pypy/module/cpyext/test/test_setobject.py
--- a/pypy/module/cpyext/test/test_setobject.py
+++ b/pypy/module/cpyext/test/test_setobject.py
@@ -1,8 +1,11 @@
+from rpython.rtyper.lltypesystem import lltype
 from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.setobject import (
     PySet_Check, PyFrozenSet_Check, PyFrozenSet_CheckExact,
     PySet_Add, PySet_Size, PySet_GET_SIZE)
+from pypy.module.cpyext.api import Py_ssize_tP, PyObjectP
+from pypy.module.cpyext.pyobject import from_ref
 
 
 class TestTupleObject(BaseApiTest):
@@ -62,6 +65,42 @@
         """)
         assert api.PyAnySet_Check(w_instance)
 
+    def test_pyset_next(self, space, api):
+        w_set = space.call_function(space.w_set, space.newtext("ab"))
+        with lltype.scoped_alloc(Py_ssize_tP.TO, 1) as pos_p:
+            with lltype.scoped_alloc(PyObjectP.TO, 1) as result_p:
+                pos_p[0] = 0
+                res = api._PySet_Next(w_set, pos_p, result_p)
+                assert res == 1
+                letter1 = space.text_w(from_ref(space, result_p[0]))
+                res = api._PySet_Next(w_set, pos_p, result_p)
+                assert res == 1
+                letter2 = space.text_w(from_ref(space, result_p[0]))
+                res = api._PySet_Next(w_set, pos_p, result_p)
+                assert res == 0
+        assert set([letter1, letter2]) == set("ab")
+
+    def test_pyset_nextentry(self, space, api):
+        w_set = space.call_function(space.w_set, space.newtext("ab"))
+        with lltype.scoped_alloc(Py_ssize_tP.TO, 1) as pos_p:
+            with lltype.scoped_alloc(PyObjectP.TO, 1) as result_p:
+                with lltype.scoped_alloc(Py_ssize_tP.TO, 1) as hash_p:
+                    pos_p[0] = 0
+                    res = api._PySet_NextEntry(w_set, pos_p, result_p, hash_p)
+                    assert res == 1
+                    w_obj = from_ref(space, result_p[0])
+                    letter1 = space.text_w(w_obj)
+                    assert hash_p[0] == space.hash_w(w_obj)
+                    res = api._PySet_NextEntry(w_set, pos_p, result_p, hash_p)
+                    assert res == 1
+                    w_obj = from_ref(space, result_p[0])
+                    letter2 = space.text_w(w_obj)
+                    assert hash_p[0] == space.hash_w(w_obj)
+                    res = api._PySet_NextEntry(w_set, pos_p, result_p, hash_p)
+                    assert res == 0
+        assert set([letter1, letter2]) == set("ab")
+
+
 class AppTestSetObject(AppTestCpythonExtensionBase):
     def test_set_macro_cast(self):
         module = self.import_extension('foo', [
diff --git a/pypy/module/cpyext/userslot.py b/pypy/module/cpyext/userslot.py
--- a/pypy/module/cpyext/userslot.py
+++ b/pypy/module/cpyext/userslot.py
@@ -22,7 +22,7 @@
 
 @slot_function([PyObject], lltype.Signed, error=-1)
 def slot_tp_hash(space, w_obj):
-    return space.int_w(space.hash(w_obj))
+    return space.hash_w(w_obj)
 
 @slot_function([PyObject, Py_ssize_t], PyObject)
 def slot_sq_item(space, w_obj, index):

From pypy.commits at gmail.com  Thu Oct 31 12:13:34 2019
From: pypy.commits at gmail.com (arigo)
Date: Thu, 31 Oct 2019 09:13:34 -0700 (PDT)
Subject: [pypy-commit] pypy _PySet_NextEntry: abandon this py3.6-based
 branch, done in default in 4748c02be810
Message-ID: <5dbb082e.1c69fb81.730c0.8744@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: _PySet_NextEntry
Changeset: r97909:c3a4841ed936
Date: 2019-10-31 17:12 +0100
http://bitbucket.org/pypy/pypy/changeset/c3a4841ed936/

Log:	abandon this py3.6-based branch, done in default in 4748c02be810


From pypy.commits at gmail.com  Thu Oct 31 13:34:55 2019
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 31 Oct 2019 10:34:55 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: Fix PyFrame._guess_function_name_parens()
Message-ID: <5dbb1b3f.1c69fb81.82035.dac9@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6
Changeset: r97910:66ff7d1e3593
Date: 2019-10-31 17:34 +0000
http://bitbucket.org/pypy/pypy/changeset/66ff7d1e3593/

Log:	Fix PyFrame._guess_function_name_parens()

diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -536,7 +536,7 @@
             return w_function.name + '()'
         if isinstance(w_function, Method):
             return self._guess_function_name_parens(None, w_function.w_function)
-        return w_function.getname(self.space) + ' object'
+        return self.space.type(w_function).getname(self.space) + ' object'
 
     def make_arguments(self, nargs, methodcall=False, w_function=None, fnname=None):
         fnname_parens = self._guess_function_name_parens(fnname, w_function)
diff --git a/pypy/interpreter/test/apptest_function.py b/pypy/interpreter/test/apptest_function.py
--- a/pypy/interpreter/test/apptest_function.py
+++ b/pypy/interpreter/test/apptest_function.py
@@ -542,6 +542,25 @@
     im = MethodType(A(), 3)
     assert list(map(im, [4])) == [7]
 
+
+class CallableBadGetattr:
+    def __getattr__(self, name):
+        # Ensure that __getattr__ doesn't get called
+        raise RuntimeError
+
+    def __call__(self, a, b, c):
+        return a, b, c
+
+def test_custom_callable_errors():
+    fn = CallableBadGetattr()
+    with raises(TypeError) as excinfo:
+        fn(*1)
+    assert excinfo.value.args[0].startswith('CallableBadGetattr object')
+    with raises(TypeError) as excinfo:
+        fn()
+    assert excinfo.value.args[0].startswith('__call__()')
+    assert fn(1, 2, 3) == (1, 2, 3)
+
 def test_invalid_creation():
     def f(): pass
     with raises(TypeError):

From pypy.commits at gmail.com  Thu Oct 31 13:38:07 2019
From: pypy.commits at gmail.com (mattip)
Date: Thu, 31 Oct 2019 10:38:07 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: merge default into py3.6
Message-ID: <5dbb1bff.1c69fb81.83871.79e6@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r97911:87d2c3bfeef2
Date: 2019-10-31 18:36 +0200
http://bitbucket.org/pypy/pypy/changeset/87d2c3bfeef2/

Log:	merge default into py3.6

diff --git a/lib_pypy/_curses_build.py b/lib_pypy/_curses_build.py
--- a/lib_pypy/_curses_build.py
+++ b/lib_pypy/_curses_build.py
@@ -1,26 +1,27 @@
+from cffi import FFI
 import os
-from cffi import FFI, VerificationError
 
-
-def find_curses_library():
-    for curses_library in ['ncursesw', 'ncurses']:
+def find_library(options):
+    for library in options:
         ffi = FFI()
-        ffi.set_source("_curses_cffi_check", "", libraries=[curses_library])
+        ffi.set_source("_curses_cffi_check", "", libraries=[library])
         try:
             ffi.compile()
         except VerificationError as e:
             e_last = e
             continue
         else:
-            return curses_library
+            return library
 
-    # If none of the libraries is available, present the user a meaningful
+    # If none of the options is available, present the user a meaningful
     # error message
     raise e_last
 
 def find_curses_include_dirs():
+    if os.path.exists('/usr/include/ncurses'):
+        return ['/usr/include/ncurses']
     if os.path.exists('/usr/include/ncursesw'):
-        return ['/usr/include/ncursesw']
+    	return ['/usr/include/ncursesw']
     return []
 
 
@@ -72,7 +73,8 @@
 void _m_getsyx(int *yx) {
     getsyx(yx[0], yx[1]);
 }
-""", libraries=[find_curses_library(), 'panel'],
+""", libraries=[find_library(['ncurses', 'ncursesw']),
+                find_library(['panel', 'panelw'])],
      include_dirs=find_curses_include_dirs())
 
 
diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -278,13 +278,14 @@
     py_dict = rffi.cast(PyDictObject, py_obj)
     if pos == 0:
         # Store the current keys in the PyDictObject.
-        decref(space, py_dict.c__tmpkeys)
         w_keyview = space.call_method(space.w_dict, "keys", w_dict)
         # w_keys must use the object strategy in order to keep the keys alive
         w_keys = space.newlist(space.listview(w_keyview))
         w_keys.switch_to_object_strategy()
+        oldkeys = py_dict.c__tmpkeys
         py_dict.c__tmpkeys = create_ref(space, w_keys)
         incref(space, py_dict.c__tmpkeys)
+        decref(space, oldkeys)
     else:
         if not py_dict.c__tmpkeys:
             # pos should have been 0, cannot fail so return 0
diff --git a/pypy/module/cpyext/include/Python.h b/pypy/module/cpyext/include/Python.h
--- a/pypy/module/cpyext/include/Python.h
+++ b/pypy/module/cpyext/include/Python.h
@@ -109,6 +109,7 @@
 #include "tupleobject.h"
 #include "dictobject.h"
 #include "longobject.h"
+#include "setobject.h"
 #include "listobject.h"
 #include "longobject.h"
 #include "unicodeobject.h"
diff --git a/pypy/module/cpyext/include/setobject.h b/pypy/module/cpyext/include/setobject.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/include/setobject.h
@@ -0,0 +1,19 @@
+
+/* set object interface */
+
+#ifndef Py_SETOBJECT_H
+#define Py_SETOBJECT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+    PyObject_HEAD
+    PyObject *_tmplist; /* a private place to put values during _PySet_Next */
+} PySetObject;
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_SETOBJECT_H */
+
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -362,11 +362,11 @@
     """
     Compute and return the hash value of an object o.  On failure, return -1.
     This is the equivalent of the Python expression hash(o)."""
-    return space.int_w(space.hash(w_obj))
+    return space.hash_w(w_obj)
 
 @cpython_api([rffi.DOUBLE], rffi.LONG, error=-1)
 def _Py_HashDouble(space, v):
-    return space.int_w(space.hash(space.newfloat(v)))
+    return space.hash_w(space.newfloat(v))
 
 @cpython_api([PyObject], lltype.Signed, error=-1)
 def PyObject_HashNotImplemented(space, o):
diff --git a/pypy/module/cpyext/setobject.py b/pypy/module/cpyext/setobject.py
--- a/pypy/module/cpyext/setobject.py
+++ b/pypy/module/cpyext/setobject.py
@@ -1,12 +1,45 @@
 from pypy.interpreter.error import OperationError, oefmt
 from rpython.rtyper.lltypesystem import rffi, lltype
-from pypy.module.cpyext.api import (cpython_api, Py_ssize_t, CANNOT_FAIL,
-                                    build_type_checkers)
+from pypy.module.cpyext.api import (
+    cpython_api, Py_ssize_t, Py_ssize_tP, CANNOT_FAIL, build_type_checkers,
+    PyObjectFields, cpython_struct, bootstrap_function, slot_function)
 from pypy.module.cpyext.pyobject import (PyObject, PyObjectP,
-    make_ref, from_ref)
+    make_ref, from_ref, as_pyobj, create_ref, make_typedescr, incref, decref)
+from pypy.module.cpyext.object import _dealloc
 from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall
 from pypy.objspace.std.setobject import W_SetObject, W_FrozensetObject, newset
 
+PySetObjectStruct = lltype.ForwardReference()
+PySetObject = lltype.Ptr(PySetObjectStruct)
+PySetObjectFields = PyObjectFields + \
+    (("_tmplist", PyObject),)
+cpython_struct("PySetObject", PySetObjectFields, PySetObjectStruct)
+
+ at bootstrap_function
+def init_setobject(space):
+    "Type description of PySetObject"
+    make_typedescr(space.w_set.layout.typedef,
+                   basestruct=PySetObject.TO,
+                   attach=set_attach,
+                   dealloc=set_dealloc)
+    make_typedescr(space.w_frozenset.layout.typedef,   # same as 'set'
+                   basestruct=PySetObject.TO,
+                   attach=set_attach,
+                   dealloc=set_dealloc)
+
+def set_attach(space, py_obj, w_obj, w_userdata=None):
+    """
+    Fills a newly allocated PySetObject with the given set object.
+    """
+    py_set = rffi.cast(PySetObject, py_obj)
+    py_set.c__tmplist = lltype.nullptr(PyObject.TO)
+
+ at slot_function([PyObject], lltype.Void)
+def set_dealloc(space, py_obj):
+    py_set = rffi.cast(PySetObject, py_obj)
+    decref(space, py_set.c__tmplist)
+    py_set.c__tmplist = lltype.nullptr(PyObject.TO)
+    _dealloc(space, py_obj)
 
 PySet_Check, PySet_CheckExact = build_type_checkers("Set")
 PyFrozenSet_Check, PyFrozenSet_CheckExact = build_type_checkers("FrozenSet")
@@ -23,7 +56,7 @@
     """Return true if obj is a set object or a frozenset object but
     not an instance of a subtype."""
     w_obj_type = space.type(w_obj)
-    return (space.is_w(w_obj_type, space.gettypefor(W_SetObject)) or 
+    return (space.is_w(w_obj_type, space.gettypefor(W_SetObject)) or
             space.is_w(w_obj_type, space.gettypefor(W_FrozensetObject)))
 
 @cpython_api([PyObject], PyObject)
@@ -126,4 +159,43 @@
     else:
         return space.call_function(space.w_frozenset, w_iterable)
 
+ at cpython_api([PyObject, Py_ssize_tP, PyObjectP, Py_ssize_tP], rffi.INT_real, error=-1)
+def _PySet_NextEntry(space, w_set, ppos, pkey, phash):
+    if w_set is None or not PyAnySet_Check(space, w_set):
+        PyErr_BadInternalCall(space)
+        return -1
+    if not pkey:
+        PyErr_BadInternalCall(space)
+        return -1
+    pos = ppos[0]
+    py_obj = as_pyobj(space, w_set)
+    py_set = rffi.cast(PySetObject, py_obj)
+    if pos == 0:
+        # Store the current item list in the PySetObject.
+        # w_keys must use the object strategy in order to keep the keys alive
+        w_keys = space.newlist(space.listview(w_set))
+        w_keys.switch_to_object_strategy()
+        oldlist = py_set.c__tmplist
+        py_set.c__tmplist = create_ref(space, w_keys)
+        incref(space, py_set.c__tmplist)
+        decref(space, oldlist)
+    else:
+        if not py_set.c__tmplist:
+            # pos should have been 0, cannot fail so return 0
+            return 0;
+        w_keys = from_ref(space, py_set.c__tmplist)
+    ppos[0] += 1
+    if pos >= space.len_w(w_keys):
+        decref(space, py_set.c__tmplist)
+        py_set.c__tmplist = lltype.nullptr(PyObject.TO)
+        return 0
+    w_key = space.listview(w_keys)[pos]
+    pkey[0] = as_pyobj(space, w_key)
+    if phash:
+        phash[0] = space.hash_w(w_key)
+    return 1
 
+ at cpython_api([PyObject, Py_ssize_tP, PyObjectP], rffi.INT_real, error=-1)
+def _PySet_Next(space, w_set, ppos, pkey):
+    null = lltype.nullptr(Py_ssize_tP.TO)
+    return _PySet_NextEntry(space, w_set, ppos, pkey, null)
diff --git a/pypy/module/cpyext/test/test_setobject.py b/pypy/module/cpyext/test/test_setobject.py
--- a/pypy/module/cpyext/test/test_setobject.py
+++ b/pypy/module/cpyext/test/test_setobject.py
@@ -1,8 +1,11 @@
+from rpython.rtyper.lltypesystem import lltype
 from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.setobject import (
     PySet_Check, PyFrozenSet_Check, PyFrozenSet_CheckExact,
     PySet_Add, PySet_Size, PySet_GET_SIZE)
+from pypy.module.cpyext.api import Py_ssize_tP, PyObjectP
+from pypy.module.cpyext.pyobject import from_ref
 
 
 class TestTupleObject(BaseApiTest):
@@ -62,6 +65,42 @@
         """)
         assert api.PyAnySet_Check(w_instance)
 
+    def test_pyset_next(self, space, api):
+        w_set = space.call_function(space.w_set, space.newtext("ab"))
+        with lltype.scoped_alloc(Py_ssize_tP.TO, 1) as pos_p:
+            with lltype.scoped_alloc(PyObjectP.TO, 1) as result_p:
+                pos_p[0] = 0
+                res = api._PySet_Next(w_set, pos_p, result_p)
+                assert res == 1
+                letter1 = space.text_w(from_ref(space, result_p[0]))
+                res = api._PySet_Next(w_set, pos_p, result_p)
+                assert res == 1
+                letter2 = space.text_w(from_ref(space, result_p[0]))
+                res = api._PySet_Next(w_set, pos_p, result_p)
+                assert res == 0
+        assert set([letter1, letter2]) == set("ab")
+
+    def test_pyset_nextentry(self, space, api):
+        w_set = space.call_function(space.w_set, space.newtext("ab"))
+        with lltype.scoped_alloc(Py_ssize_tP.TO, 1) as pos_p:
+            with lltype.scoped_alloc(PyObjectP.TO, 1) as result_p:
+                with lltype.scoped_alloc(Py_ssize_tP.TO, 1) as hash_p:
+                    pos_p[0] = 0
+                    res = api._PySet_NextEntry(w_set, pos_p, result_p, hash_p)
+                    assert res == 1
+                    w_obj = from_ref(space, result_p[0])
+                    letter1 = space.text_w(w_obj)
+                    assert hash_p[0] == space.hash_w(w_obj)
+                    res = api._PySet_NextEntry(w_set, pos_p, result_p, hash_p)
+                    assert res == 1
+                    w_obj = from_ref(space, result_p[0])
+                    letter2 = space.text_w(w_obj)
+                    assert hash_p[0] == space.hash_w(w_obj)
+                    res = api._PySet_NextEntry(w_set, pos_p, result_p, hash_p)
+                    assert res == 0
+        assert set([letter1, letter2]) == set("ab")
+
+
 class AppTestSetObject(AppTestCpythonExtensionBase):
     def test_set_macro_cast(self):
         module = self.import_extension('foo', [
diff --git a/pypy/module/cpyext/userslot.py b/pypy/module/cpyext/userslot.py
--- a/pypy/module/cpyext/userslot.py
+++ b/pypy/module/cpyext/userslot.py
@@ -22,7 +22,7 @@
 
 @slot_function([PyObject], lltype.Signed, error=-1)
 def slot_tp_hash(space, w_obj):
-    return space.int_w(space.hash(w_obj))
+    return space.hash_w(w_obj)
 
 @slot_function([PyObject, Py_ssize_t], PyObject)
 def slot_sq_item(space, w_obj, index):

From pypy.commits at gmail.com  Thu Oct 31 13:38:10 2019
From: pypy.commits at gmail.com (mattip)
Date: Thu, 31 Oct 2019 10:38:10 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: merge heads
Message-ID: <5dbb1c02.1c69fb81.759a6.8460@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r97912:33fe0dcb31b8
Date: 2019-10-31 19:37 +0200
http://bitbucket.org/pypy/pypy/changeset/33fe0dcb31b8/

Log:	merge heads

diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -536,7 +536,7 @@
             return w_function.name + '()'
         if isinstance(w_function, Method):
             return self._guess_function_name_parens(None, w_function.w_function)
-        return w_function.getname(self.space) + ' object'
+        return self.space.type(w_function).getname(self.space) + ' object'
 
     def make_arguments(self, nargs, methodcall=False, w_function=None, fnname=None):
         fnname_parens = self._guess_function_name_parens(fnname, w_function)
diff --git a/pypy/interpreter/test/apptest_function.py b/pypy/interpreter/test/apptest_function.py
--- a/pypy/interpreter/test/apptest_function.py
+++ b/pypy/interpreter/test/apptest_function.py
@@ -542,6 +542,25 @@
     im = MethodType(A(), 3)
     assert list(map(im, [4])) == [7]
 
+
+class CallableBadGetattr:
+    def __getattr__(self, name):
+        # Ensure that __getattr__ doesn't get called
+        raise RuntimeError
+
+    def __call__(self, a, b, c):
+        return a, b, c
+
+def test_custom_callable_errors():
+    fn = CallableBadGetattr()
+    with raises(TypeError) as excinfo:
+        fn(*1)
+    assert excinfo.value.args[0].startswith('CallableBadGetattr object')
+    with raises(TypeError) as excinfo:
+        fn()
+    assert excinfo.value.args[0].startswith('__call__()')
+    assert fn(1, 2, 3) == (1, 2, 3)
+
 def test_invalid_creation():
     def f(): pass
     with raises(TypeError):

From pypy.commits at gmail.com  Thu Oct 31 13:42:06 2019
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 31 Oct 2019 10:42:06 -0700 (PDT)
Subject: [pypy-commit] pypy py3.7: Merged in
 Yannick_Jadoul/pypy/py3.7-bpo-31333 (pull request #675)
Message-ID: <5dbb1cee.1c69fb81.83871.7ba4@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.7
Changeset: r97918:4a29e3fa9b40
Date: 2019-10-31 17:41 +0000
http://bitbucket.org/pypy/pypy/changeset/4a29e3fa9b40/

Log:	Merged in Yannick_Jadoul/pypy/py3.7-bpo-31333 (pull request #675)

	bpo-31333: Adding built-in module _abc

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -27,7 +27,7 @@
     "_codecs", "atexit", "gc", "_weakref", "marshal", "errno", "imp",
     "itertools", "math", "cmath", "_sre", "_pickle_support",
     "parser", "symbol", "token", "_ast", "_random", "__pypy__",
-    "_string", "_testing", "time"
+    "_string", "_testing", "time", "_abc",
 ])
 
 
diff --git a/pypy/module/_abc/__init__.py b/pypy/module/_abc/__init__.py
new file mode 100644
diff --git a/pypy/module/_abc/app_abc.py b/pypy/module/_abc/app_abc.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_abc/app_abc.py
@@ -0,0 +1,175 @@
+# NOT_RPYTHON
+"""
+Plain Python definition of the builtin ABC-related functions.
+"""
+
+# Cannot use _weakrefset like in _py_abc.py, since it is not a built-in module
+from _weakref import ref
+
+# We'll make our own WeakSet instead, with only the functionality that's needed
+# This replaces the easily-forgetable calls to '_add_to_weak_set' and
+# '_in_weak_set' from the CPython implementation
+class SimpleWeakSet:
+    def __init__(self, data=None):
+        self.data = set()
+        def _remove(item, selfref=ref(self)):
+            self = selfref()
+            if self is not None:
+                self.data.discard(item)
+        self._remove = _remove
+
+    def __iter__(self):
+        # Weakref callback may remove entry from set.
+        # So we make a copy first.
+        copy = list(self.data)
+        for itemref in copy:
+            item = itemref()
+            yield item
+
+    def __contains__(self, item):
+        try:
+            wr = ref(item)
+        except TypeError:
+            return False
+        return wr in self.data
+
+    def add(self, item):
+        self.data.add(ref(item, self._remove))
+
+    def clear(self):
+        self.data.clear()
+
+
+abc_invalidation_counter = 0
+
+
+def get_cache_token():
+    """Returns the current ABC cache token.
+
+    The token is an opaque object (supporting equality testing) identifying the
+    current version of the ABC cache for virtual subclasses. The token changes
+    with every call to ``register()`` on any ABC.
+    """
+    return abc_invalidation_counter
+
+
+def _abc_init(cls):
+    """Internal ABC helper for class set-up. Should be never used outside abc module."""
+    namespace = cls.__dict__
+    abstracts = {name
+                for name, value in namespace.items()
+                if getattr(value, "__isabstractmethod__", False)}
+    bases = cls.__bases__
+    for base in bases:
+        for name in getattr(base, "__abstractmethods__", set()):
+            value = getattr(cls, name, None)
+            if getattr(value, "__isabstractmethod__", False):
+                abstracts.add(name)
+    cls.__abstractmethods__ = frozenset(abstracts)
+    # Set up inheritance registry
+    cls._abc_registry = SimpleWeakSet()
+    cls._abc_cache = SimpleWeakSet()
+    cls._abc_negative_cache = SimpleWeakSet()
+    cls._abc_negative_cache_version = abc_invalidation_counter
+
+
+def _abc_register(cls, subclass):
+    """Internal ABC helper for subclasss registration. Should be never used outside abc module."""
+    if not isinstance(subclass, type):
+        raise TypeError("Can only register classes")
+    if issubclass(subclass, cls):
+        return subclass  # Already a subclass
+    # Subtle: test for cycles *after* testing for "already a subclass";
+    # this means we allow X.register(X) and interpret it as a no-op.
+    if issubclass(cls, subclass):
+        # This would create a cycle, which is bad for the algorithm below
+        raise RuntimeError("Refusing to create an inheritance cycle")
+    cls._abc_registry.add(subclass)
+    global abc_invalidation_counter
+    abc_invalidation_counter += 1  # Invalidate negative cache
+    return subclass
+
+
+def _abc_instancecheck(cls, instance):
+    """Internal ABC helper for instance checks. Should be never used outside abc module."""
+    # Inline the cache checking
+    subclass = instance.__class__
+    if subclass in cls._abc_cache:
+        return True
+    subtype = type(instance)
+    if subtype is subclass:
+        if (cls._abc_negative_cache_version == abc_invalidation_counter and
+            subclass in cls._abc_negative_cache):
+            return False
+        # Fall back to the subclass check.
+        return cls.__subclasscheck__(subclass)
+    return any(cls.__subclasscheck__(c) for c in (subclass, subtype))
+
+
+def _abc_subclasscheck(cls, subclass):
+    """Internal ABC helper for subclasss checks. Should be never used outside abc module."""
+    if not isinstance(subclass, type):
+        raise TypeError('issubclass() arg 1 must be a class')
+    # Check cache
+    if subclass in cls._abc_cache:
+        return True
+    # Check negative cache; may have to invalidate
+    if cls._abc_negative_cache_version < abc_invalidation_counter:
+        # Invalidate the negative cache
+        cls._abc_negative_cache = SimpleWeakSet()
+        cls._abc_negative_cache_version = abc_invalidation_counter
+    elif subclass in cls._abc_negative_cache:
+        return False
+    # Check the subclass hook
+    ok = cls.__subclasshook__(subclass)
+    if ok is not NotImplemented:
+        assert isinstance(ok, bool)
+        if ok:
+            cls._abc_cache.add(subclass)
+        else:
+            cls._abc_negative_cache.add(subclass)
+        return ok
+    # Check if it's a direct subclass
+    if cls in getattr(subclass, '__mro__', ()):
+        cls._abc_cache.add(subclass)
+        return True
+    # Check if it's a subclass of a registered class (recursive)
+    for rcls in cls._abc_registry:
+        if issubclass(subclass, rcls):
+            cls._abc_cache.add(subclass)
+            return True
+    # Check if it's a subclass of a subclass (recursive)
+    for scls in cls.__subclasses__():
+        if issubclass(subclass, scls):
+            cls._abc_cache.add(subclass)
+            return True
+    # No dice; update negative cache
+    cls._abc_negative_cache.add(subclass)
+    return False
+
+
+def _get_dump(cls):
+    """Internal ABC helper for cache and registry debugging
+
+    Return shallow copies of registry, of both caches, and
+    negative cache version. Don't call this function directly,
+    instead use ABC._dump_registry() for a nice repr."""
+    return (cls._abc_registry.data,
+            cls._abc_cache.data,
+            cls._abc_negative_cache.data,
+            cls._abc_negative_cache_version)
+
+
+def _reset_registry(cls):
+    """Internal ABC helper to reset registry of a given class.
+
+    Should be only used by refleak.py"""
+    cls._abc_registry.clear()
+
+
+def _reset_caches(cls):
+    """Internal ABC helper to reset both caches of a given class.
+
+    Should be only used by refleak.py"""
+    cls._abc_cache.clear()
+    cls._abc_negative_cache.clear()
diff --git a/pypy/module/_abc/moduledef.py b/pypy/module/_abc/moduledef.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_abc/moduledef.py
@@ -0,0 +1,17 @@
+from pypy.interpreter.mixedmodule import MixedModule
+
+class Module(MixedModule):
+
+    appleveldefs = {
+        "get_cache_token": "app_abc.get_cache_token",
+        "_abc_init": "app_abc._abc_init",
+        "_abc_register": "app_abc._abc_register",
+        "_abc_instancecheck": "app_abc._abc_instancecheck",
+        "_abc_subclasscheck": "app_abc._abc_subclasscheck",
+        "_get_dump": "app_abc._get_dump",
+        "_reset_registry": "app_abc._reset_registry",
+        "_reset_caches": "app_abc._reset_caches",
+    }
+
+    interpleveldefs = {
+    }
diff --git a/pypy/module/_abc/test/test_abc.py b/pypy/module/_abc/test/test_abc.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_abc/test/test_abc.py
@@ -0,0 +1,84 @@
+class AppTestAbcModule:
+    def test_import_builtin(self):
+        from _abc import (get_cache_token, _abc_init, _abc_register,
+                          _abc_instancecheck, _abc_subclasscheck, _get_dump,
+                          _reset_registry, _reset_caches)
+
+    def test_basic(self):
+        import _abc
+
+        class SomeABC: pass
+        
+        _abc._abc_init(SomeABC)
+        assert hasattr(SomeABC, '__abstractmethods__')
+
+        class SomeConcreteSubClass: pass
+        _abc._abc_register(SomeABC, SomeConcreteSubClass)
+        
+        # _abc._abc_instancecheck calls cls.__subclasscheck__, but since
+        # we've only called _abc_init and haven't set the metaclass, we
+        # need to monkeypatch SomeABC before calling _abc_instancecheck
+        from types import MethodType
+        SomeABC.__subclasscheck__ = MethodType(_abc._abc_subclasscheck, SomeABC)
+        assert _abc._abc_instancecheck(SomeABC, SomeConcreteSubClass())
+        assert _abc._abc_subclasscheck(SomeABC, SomeConcreteSubClass)
+
+        class SomeOtherClass: pass
+        assert not _abc._abc_instancecheck(SomeABC, SomeOtherClass())
+        assert not _abc._abc_subclasscheck(SomeABC, SomeOtherClass)
+
+        _abc._reset_registry(SomeABC)
+        _abc._reset_caches(SomeABC)
+        assert not _abc._abc_instancecheck(SomeABC, SomeConcreteSubClass())
+        assert not _abc._abc_subclasscheck(SomeABC, SomeConcreteSubClass)
+    
+    def test_cache(self):
+        import _abc
+
+        class SomeABC:
+            pass
+
+        token_before = _abc.get_cache_token()
+        assert _abc.get_cache_token() == token_before
+
+        _abc._abc_init(SomeABC)
+        assert _abc.get_cache_token() == token_before
+
+        class SomeConcreteSubClass: pass
+        _abc._abc_register(SomeABC, SomeConcreteSubClass)
+        assert _abc.get_cache_token() != token_before
+
+        registry, cache, negative_cache, negative_cache_version = _abc._get_dump(SomeABC)
+        assert len(registry) == 1
+        assert len(cache) == 0
+        assert len(negative_cache) == 0
+        assert negative_cache_version == token_before
+
+        assert _abc._abc_subclasscheck(SomeABC, SomeConcreteSubClass)
+        registry, cache, negative_cache, negative_cache_version = _abc._get_dump(SomeABC)
+        assert len(registry) == 1
+        assert len(cache) == 1
+        assert len(negative_cache) == 0
+        assert negative_cache_version == _abc.get_cache_token()
+
+        class SomeOtherClass: pass
+        assert not _abc._abc_subclasscheck(SomeABC, SomeOtherClass)
+        registry, cache, negative_cache, negative_cache_version = _abc._get_dump(SomeABC)
+        assert len(registry) == 1
+        assert len(cache) == 1
+        assert len(negative_cache) == 1
+        assert negative_cache_version == _abc.get_cache_token()
+
+        _abc._reset_caches(SomeABC)
+        registry, cache, negative_cache, negative_cache_version = _abc._get_dump(SomeABC)
+        assert len(registry) == 1
+        assert len(cache) == 0
+        assert len(negative_cache) == 0
+        assert negative_cache_version == _abc.get_cache_token()
+
+        _abc._reset_registry(SomeABC)
+        registry, cache, negative_cache, negative_cache_version = _abc._get_dump(SomeABC)
+        assert len(registry) == 0
+        assert len(cache) == 0
+        assert len(negative_cache) == 0
+        assert negative_cache_version == _abc.get_cache_token()

From pypy.commits at gmail.com  Thu Oct 31 13:42:24 2019
From: pypy.commits at gmail.com (Yannick_Jadoul)
Date: Thu, 31 Oct 2019 10:42:24 -0700 (PDT)
Subject: [pypy-commit] pypy py3.7-bpo-31333: Adding built-in module _abc
Message-ID: <5dbb1d00.1c69fb81.64e14.1922@mx.google.com>

Author: Yannick Jadoul <yannick.jadoul at belgacom.net>
Branch: py3.7-bpo-31333
Changeset: r97913:1afc6d0f0699
Date: 2019-10-18 19:58 +0200
http://bitbucket.org/pypy/pypy/changeset/1afc6d0f0699/

Log:	Adding built-in module _abc

diff --git a/pypy/module/_abc/__init__.py b/pypy/module/_abc/__init__.py
new file mode 100644
diff --git a/pypy/module/_abc/app_abc.py b/pypy/module/_abc/app_abc.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_abc/app_abc.py
@@ -0,0 +1,139 @@
+# NOT_RPYTHON
+"""
+Plain Python definition of the builtin ABC-related functions.
+"""
+
+from _weakrefset import WeakSet
+
+
+_abc_invalidation_counter = 0
+
+
+def get_cache_token():
+    """Returns the current ABC cache token.
+
+    The token is an opaque object (supporting equality testing) identifying the
+    current version of the ABC cache for virtual subclasses. The token changes
+    with every call to ``register()`` on any ABC.
+    """
+    return _abc_invalidation_counter
+
+
+def _abc_init(cls):
+    """Internal ABC helper for class set-up. Should be never used outside abc module."""
+    namespace = cls.__dict__
+    abstracts = {name
+                for name, value in namespace.items()
+                if getattr(value, "__isabstractmethod__", False)}
+    bases = cls.__bases__
+    for base in bases:
+        for name in getattr(base, "__abstractmethods__", set()):
+            value = getattr(cls, name, None)
+            if getattr(value, "__isabstractmethod__", False):
+                abstracts.add(name)
+    cls.__abstractmethods__ = frozenset(abstracts)
+    # Set up inheritance registry
+    cls._abc_registry = WeakSet()
+    cls._abc_cache = WeakSet()
+    cls._abc_negative_cache = WeakSet()
+    cls._abc_negative_cache_version = _abc_invalidation_counter
+
+
+def _abc_register(cls, subclass):
+    """Internal ABC helper for subclasss registration. Should be never used outside abc module."""
+    if not isinstance(subclass, type):
+        raise TypeError("Can only register classes")
+    if issubclass(subclass, cls):
+        return subclass  # Already a subclass
+    # Subtle: test for cycles *after* testing for "already a subclass";
+    # this means we allow X.register(X) and interpret it as a no-op.
+    if issubclass(cls, subclass):
+        # This would create a cycle, which is bad for the algorithm below
+        raise RuntimeError("Refusing to create an inheritance cycle")
+    cls._abc_registry.add(subclass)
+    global _abc_invalidation_counter
+    _abc_invalidation_counter += 1  # Invalidate negative cache
+    return subclass
+
+
+def _abc_instancecheck(cls, instance):
+    """Internal ABC helper for instance checks. Should be never used outside abc module."""
+    # Inline the cache checking
+    subclass = instance.__class__
+    if subclass in cls._abc_cache:
+        return True
+    subtype = type(instance)
+    if subtype is subclass:
+        if (cls._abc_negative_cache_version ==
+            _abc_invalidation_counter and
+            subclass in cls._abc_negative_cache):
+            return False
+        # Fall back to the subclass check.
+        return cls.__subclasscheck__(subclass)
+    return any(cls.__subclasscheck__(c) for c in (subclass, subtype))
+
+
+def _abc_subclasscheck(cls, subclass):
+    """Internal ABC helper for subclasss checks. Should be never used outside abc module."""
+    if not isinstance(subclass, type):
+        raise TypeError('issubclass() arg 1 must be a class')
+    # Check cache
+    if subclass in cls._abc_cache:
+        return True
+    # Check negative cache; may have to invalidate
+    if cls._abc_negative_cache_version < _abc_invalidation_counter:
+        # Invalidate the negative cache
+        cls._abc_negative_cache = WeakSet()
+        cls._abc_negative_cache_version = _abc_invalidation_counter
+    elif subclass in cls._abc_negative_cache:
+        return False
+    # Check the subclass hook
+    ok = cls.__subclasshook__(subclass)
+    if ok is not NotImplemented:
+        assert isinstance(ok, bool)
+        if ok:
+            cls._abc_cache.add(subclass)
+        else:
+            cls._abc_negative_cache.add(subclass)
+        return ok
+    # Check if it's a direct subclass
+    if cls in getattr(subclass, '__mro__', ()):
+        cls._abc_cache.add(subclass)
+        return True
+    # Check if it's a subclass of a registered class (recursive)
+    for rcls in cls._abc_registry:
+        if issubclass(subclass, rcls):
+            cls._abc_cache.add(subclass)
+            return True
+    # Check if it's a subclass of a subclass (recursive)
+    for scls in cls.__subclasses__():
+        if issubclass(subclass, scls):
+            cls._abc_cache.add(subclass)
+            return True
+    # No dice; update negative cache
+    cls._abc_negative_cache.add(subclass)
+    return False
+
+
+def _get_dump(cls):
+    """Internal ABC helper for cache and registry debugging
+
+    Return shallow copies of registry, of both caches, and
+    negative cache version. Don't call this function directly,
+    instead use ABC._dump_registry() for a nice repr."""
+    return (cls._abc_registry, cls._abc_cache, cls._abc_negative_cache, cls._abc_negative_cache_version)
+
+
+def _reset_registry(cls):
+    """Internal ABC helper to reset registry of a given class.
+
+    Should be only used by refleak.py"""
+    cls._abc_registry.clear()
+
+
+def _reset_caches(cls):
+    """Internal ABC helper to reset both caches of a given class.
+
+    Should be only used by refleak.py"""
+    cls._abc_cache.clear()
+    cls._abc_negative_cache.clear()
diff --git a/pypy/module/_abc/moduledef.py b/pypy/module/_abc/moduledef.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_abc/moduledef.py
@@ -0,0 +1,17 @@
+from pypy.interpreter.mixedmodule import MixedModule
+
+class Module(MixedModule):
+
+    appleveldefs = {
+        "get_cache_token": "app_abc.get_cache_token",
+        "_abc_init": "app_abc._abc_init",
+        "_abc_register": "app_abc._abc_register",
+        "_abc_instancecheck": "app_abc._abc_instancecheck",
+        "_abc_subclasscheck": "app_abc._abc_subclasscheck",
+        "_get_dump": "app_abc._get_dump",
+        "_reset_registry": "app_abc._reset_registry",
+        "_reset_caches": "app_abc._reset_caches",
+    }
+
+    interpleveldefs = {
+    }

From pypy.commits at gmail.com  Thu Oct 31 13:42:26 2019
From: pypy.commits at gmail.com (Yannick_Jadoul)
Date: Thu, 31 Oct 2019 10:42:26 -0700 (PDT)
Subject: [pypy-commit] pypy py3.7-bpo-31333: Not importing _weakrefset
 anymore to implement _abc module, and added '_abc' to essential_modules list
Message-ID: <5dbb1d02.1c69fb81.dacbe.ae4f@mx.google.com>

Author: Yannick Jadoul <yannick.jadoul at belgacom.net>
Branch: py3.7-bpo-31333
Changeset: r97914:4ad607fdca5d
Date: 2019-10-20 02:12 +0200
http://bitbucket.org/pypy/pypy/changeset/4ad607fdca5d/

Log:	Not importing _weakrefset anymore to implement _abc module, and
	added '_abc' to essential_modules list

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -17,7 +17,7 @@
 essential_modules = set([
     "exceptions", "_io", "sys", "builtins", "posix", "_warnings",
     "itertools", "_frozen_importlib", "operator", "_locale", "struct",
-    "thread",
+    "thread", "_abc",
 ])
 if sys.platform == "win32":
     essential_modules.add("_winreg")
diff --git a/pypy/module/_abc/app_abc.py b/pypy/module/_abc/app_abc.py
--- a/pypy/module/_abc/app_abc.py
+++ b/pypy/module/_abc/app_abc.py
@@ -3,10 +3,41 @@
 Plain Python definition of the builtin ABC-related functions.
 """
 
-from _weakrefset import WeakSet
+# Cannot use _weakrefset like in _py_abc.py, since it is not a built-in module
+from _weakref import ref
 
+# We'll make our own WeakSet instead, with only the functionality that's needed
+# This replaces the easily-forgetable calls to '_add_to_weak_set' and
+# '_in_weak_set' from the CPython implementation
+class SimpleWeakSet:
+    def __init__(self, data=None):
+        self.data = set()
+        def _remove(item, selfref=ref(self)):
+            self = selfref()
+            if self is not None:
+                self.data.discard(item)
+        self._remove = _remove
 
-_abc_invalidation_counter = 0
+    def __iter__(self):
+        # Weakref callback may remove entry from set.
+        # So we make a copy first.
+        copy = list(self.data)
+        for itemref in copy:
+            item = itemref()
+            yield item
+
+    def __contains__(self, item):
+        try:
+            wr = ref(item)
+        except TypeError:
+            return False
+        return wr in self.data
+
+    def add(self, item):
+        self.data.add(ref(item, self._remove))
+
+
+abc_invalidation_counter = 0
 
 
 def get_cache_token():
@@ -16,7 +47,7 @@
     current version of the ABC cache for virtual subclasses. The token changes
     with every call to ``register()`` on any ABC.
     """
-    return _abc_invalidation_counter
+    return abc_invalidation_counter
 
 
 def _abc_init(cls):
@@ -33,10 +64,10 @@
                 abstracts.add(name)
     cls.__abstractmethods__ = frozenset(abstracts)
     # Set up inheritance registry
-    cls._abc_registry = WeakSet()
-    cls._abc_cache = WeakSet()
-    cls._abc_negative_cache = WeakSet()
-    cls._abc_negative_cache_version = _abc_invalidation_counter
+    cls._abc_registry = SimpleWeakSet()
+    cls._abc_cache = SimpleWeakSet()
+    cls._abc_negative_cache = SimpleWeakSet()
+    cls._abc_negative_cache_version = abc_invalidation_counter
 
 
 def _abc_register(cls, subclass):
@@ -51,8 +82,8 @@
         # This would create a cycle, which is bad for the algorithm below
         raise RuntimeError("Refusing to create an inheritance cycle")
     cls._abc_registry.add(subclass)
-    global _abc_invalidation_counter
-    _abc_invalidation_counter += 1  # Invalidate negative cache
+    global abc_invalidation_counter
+    abc_invalidation_counter += 1  # Invalidate negative cache
     return subclass
 
 
@@ -64,8 +95,7 @@
         return True
     subtype = type(instance)
     if subtype is subclass:
-        if (cls._abc_negative_cache_version ==
-            _abc_invalidation_counter and
+        if (cls._abc_negative_cache_version == abc_invalidation_counter and
             subclass in cls._abc_negative_cache):
             return False
         # Fall back to the subclass check.
@@ -81,10 +111,10 @@
     if subclass in cls._abc_cache:
         return True
     # Check negative cache; may have to invalidate
-    if cls._abc_negative_cache_version < _abc_invalidation_counter:
+    if cls._abc_negative_cache_version < abc_invalidation_counter:
         # Invalidate the negative cache
-        cls._abc_negative_cache = WeakSet()
-        cls._abc_negative_cache_version = _abc_invalidation_counter
+        cls._abc_negative_cache = SimpleWeakSet()
+        cls._abc_negative_cache_version = abc_invalidation_counter
     elif subclass in cls._abc_negative_cache:
         return False
     # Check the subclass hook

From pypy.commits at gmail.com  Thu Oct 31 13:42:28 2019
From: pypy.commits at gmail.com (Yannick_Jadoul)
Date: Thu, 31 Oct 2019 10:42:28 -0700 (PDT)
Subject: [pypy-commit] pypy py3.7-bpo-31333: Fixing _abc._get_dump to return
 sets, rather than the new SimpleWeakSet wrapper
Message-ID: <5dbb1d04.1c69fb81.a2525.dba1@mx.google.com>

Author: Yannick Jadoul <yannick.jadoul at belgacom.net>
Branch: py3.7-bpo-31333
Changeset: r97915:8c14c8098e5a
Date: 2019-10-20 22:48 +0200
http://bitbucket.org/pypy/pypy/changeset/8c14c8098e5a/

Log:	Fixing _abc._get_dump to return sets, rather than the new
	SimpleWeakSet wrapper

diff --git a/pypy/module/_abc/app_abc.py b/pypy/module/_abc/app_abc.py
--- a/pypy/module/_abc/app_abc.py
+++ b/pypy/module/_abc/app_abc.py
@@ -151,7 +151,10 @@
     Return shallow copies of registry, of both caches, and
     negative cache version. Don't call this function directly,
     instead use ABC._dump_registry() for a nice repr."""
-    return (cls._abc_registry, cls._abc_cache, cls._abc_negative_cache, cls._abc_negative_cache_version)
+    return (cls._abc_registry.data,
+            cls._abc_cache.data,
+            cls._abc_negative_cache.data,
+            cls._abc_negative_cache_version)
 
 
 def _reset_registry(cls):

From pypy.commits at gmail.com  Thu Oct 31 13:42:31 2019
From: pypy.commits at gmail.com (Yannick_Jadoul)
Date: Thu, 31 Oct 2019 10:42:31 -0700 (PDT)
Subject: [pypy-commit] pypy py3.7-bpo-31333: Added tests and fixed
 _reset_registry and _reset_caches
Message-ID: <5dbb1d07.1c69fb81.6b5e7.a605@mx.google.com>

Author: Yannick Jadoul <yannick.jadoul at belgacom.net>
Branch: py3.7-bpo-31333
Changeset: r97916:1e2d3e37d725
Date: 2019-10-20 23:15 +0200
http://bitbucket.org/pypy/pypy/changeset/1e2d3e37d725/

Log:	Added tests and fixed _reset_registry and _reset_caches

diff --git a/pypy/module/_abc/app_abc.py b/pypy/module/_abc/app_abc.py
--- a/pypy/module/_abc/app_abc.py
+++ b/pypy/module/_abc/app_abc.py
@@ -36,6 +36,9 @@
     def add(self, item):
         self.data.add(ref(item, self._remove))
 
+    def clear(self):
+        self.data.clear()
+
 
 abc_invalidation_counter = 0
 
diff --git a/pypy/module/_abc/test/test_abc.py b/pypy/module/_abc/test/test_abc.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_abc/test/test_abc.py
@@ -0,0 +1,84 @@
+class AppTestAbcModule:
+    def test_import_builtin(self):
+        from _abc import (get_cache_token, _abc_init, _abc_register,
+                          _abc_instancecheck, _abc_subclasscheck, _get_dump,
+                          _reset_registry, _reset_caches)
+
+    def test_basic(self):
+        import _abc
+
+        class SomeABC: pass
+        
+        _abc._abc_init(SomeABC)
+        assert hasattr(SomeABC, '__abstractmethods__')
+
+        class SomeConcreteSubClass: pass
+        _abc._abc_register(SomeABC, SomeConcreteSubClass)
+        
+        # _abc._abc_instancecheck calls cls.__subclasscheck__, but since
+        # we've only called _abc_init and haven't set the metaclass, we
+        # need to monkeypatch SomeABC before calling _abc_instancecheck
+        from types import MethodType
+        SomeABC.__subclasscheck__ = MethodType(_abc._abc_subclasscheck, SomeABC)
+        assert _abc._abc_instancecheck(SomeABC, SomeConcreteSubClass())
+        assert _abc._abc_subclasscheck(SomeABC, SomeConcreteSubClass)
+
+        class SomeOtherClass: pass
+        assert not _abc._abc_instancecheck(SomeABC, SomeOtherClass())
+        assert not _abc._abc_subclasscheck(SomeABC, SomeOtherClass)
+
+        _abc._reset_registry(SomeABC)
+        _abc._reset_caches(SomeABC)
+        assert not _abc._abc_instancecheck(SomeABC, SomeConcreteSubClass())
+        assert not _abc._abc_subclasscheck(SomeABC, SomeConcreteSubClass)
+    
+    def test_cache(self):
+        import _abc
+
+        class SomeABC:
+            pass
+
+        token_before = _abc.get_cache_token()
+        assert _abc.get_cache_token() == token_before
+
+        _abc._abc_init(SomeABC)
+        assert _abc.get_cache_token() == token_before
+
+        class SomeConcreteSubClass: pass
+        _abc._abc_register(SomeABC, SomeConcreteSubClass)
+        assert _abc.get_cache_token() != token_before
+
+        registry, cache, negative_cache, negative_cache_version = _abc._get_dump(SomeABC)
+        assert len(registry) == 1
+        assert len(cache) == 0
+        assert len(negative_cache) == 0
+        assert negative_cache_version == token_before
+
+        assert _abc._abc_subclasscheck(SomeABC, SomeConcreteSubClass)
+        registry, cache, negative_cache, negative_cache_version = _abc._get_dump(SomeABC)
+        assert len(registry) == 1
+        assert len(cache) == 1
+        assert len(negative_cache) == 0
+        assert negative_cache_version == _abc.get_cache_token()
+
+        class SomeOtherClass: pass
+        assert not _abc._abc_subclasscheck(SomeABC, SomeOtherClass)
+        registry, cache, negative_cache, negative_cache_version = _abc._get_dump(SomeABC)
+        assert len(registry) == 1
+        assert len(cache) == 1
+        assert len(negative_cache) == 1
+        assert negative_cache_version == _abc.get_cache_token()
+
+        _abc._reset_caches(SomeABC)
+        registry, cache, negative_cache, negative_cache_version = _abc._get_dump(SomeABC)
+        assert len(registry) == 1
+        assert len(cache) == 0
+        assert len(negative_cache) == 0
+        assert negative_cache_version == _abc.get_cache_token()
+
+        _abc._reset_registry(SomeABC)
+        registry, cache, negative_cache, negative_cache_version = _abc._get_dump(SomeABC)
+        assert len(registry) == 0
+        assert len(cache) == 0
+        assert len(negative_cache) == 0
+        assert negative_cache_version == _abc.get_cache_token()

From pypy.commits at gmail.com  Thu Oct 31 13:42:34 2019
From: pypy.commits at gmail.com (Yannick_Jadoul)
Date: Thu, 31 Oct 2019 10:42:34 -0700 (PDT)
Subject: [pypy-commit] pypy py3.7-bpo-31333: Moving _abc from
 essential_modules to default_modules
Message-ID: <5dbb1d0a.1c69fb81.45a59.f11e@mx.google.com>

Author: Yannick Jadoul <yannick.jadoul at belgacom.net>
Branch: py3.7-bpo-31333
Changeset: r97917:cf9fa90f1848
Date: 2019-10-21 14:27 +0200
http://bitbucket.org/pypy/pypy/changeset/cf9fa90f1848/

Log:	Moving _abc from essential_modules to default_modules

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -17,7 +17,7 @@
 essential_modules = set([
     "exceptions", "_io", "sys", "builtins", "posix", "_warnings",
     "itertools", "_frozen_importlib", "operator", "_locale", "struct",
-    "thread", "_abc",
+    "thread",
 ])
 if sys.platform == "win32":
     essential_modules.add("_winreg")
@@ -27,7 +27,7 @@
     "_codecs", "atexit", "gc", "_weakref", "marshal", "errno", "imp",
     "itertools", "math", "cmath", "_sre", "_pickle_support",
     "parser", "symbol", "token", "_ast", "_random", "__pypy__",
-    "_string", "_testing", "time"
+    "_string", "_testing", "time", "_abc",
 ])
 
 

From pypy.commits at gmail.com  Thu Oct 31 14:45:26 2019
From: pypy.commits at gmail.com (mattip)
Date: Thu, 31 Oct 2019 11:45:26 -0700 (PDT)
Subject: [pypy-commit] buildbot default: add more documentation around using
 docker
Message-ID: <5dbb2bc6.1c69fb81.f0fcd.f63b@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r1099:c40bc42d1f79
Date: 2019-10-31 20:45 +0200
http://bitbucket.org/pypy/buildbot/changeset/c40bc42d1f79/

Log:	add more documentation around using docker

diff --git a/docker/Dockerfile b/docker/Dockerfile
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -14,9 +14,19 @@
 # to "hang" the console running the slave. To stop the slave, simply CTRL-C or
 # kill the process.
 #
-# To enter the buildslave image
+# You might want to keep the PYPY_USESSION_DIR where the testing/building
+# artifacts are. Docker will not do this for you, so do something like this
+# to save the files outside the docker
+#
+# mkdir -p build_dir/tmp
+# docker run --rm --user $UID -v<abspath/to/builder/dir>:/build_dir> \
+#     -ePYPY_USESSION_DIR=/build_dir/tmp buildslave
+#
+# To enter the buildslave image, add a shell command to the end
+#
 # docker run -it --user $UID -v<abspath/to/builder/dir>:/build_dir> \
-#     buildslave /bin/bash
+#     -ePYPY_USESSION_DIR=/build_dir/tmp buildslave /bin/bash
+#
 
 FROM centos:centos6
 

From pypy.commits at gmail.com  Thu Oct 31 16:14:22 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 31 Oct 2019 13:14:22 -0700 (PDT)
Subject: [pypy-commit] pypy default: test showing the problem of #3108
Message-ID: <5dbb409e.1c69fb81.268ab.f18b@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r97923:73f583091cbf
Date: 2019-10-31 21:04 +0100
http://bitbucket.org/pypy/pypy/changeset/73f583091cbf/

Log:	test showing the problem of #3108

diff --git a/pypy/objspace/std/test/test_jsondict.py b/pypy/objspace/std/test/test_jsondict.py
--- a/pypy/objspace/std/test/test_jsondict.py
+++ b/pypy/objspace/std/test/test_jsondict.py
@@ -95,3 +95,19 @@
         assert not 1 in d
         assert __pypy__.strategy(d) == "UnicodeDictStrategy"
         assert list(d) == [u"a", u"b"]
+
+    def test_bug(self):
+        import _pypyjson
+        a =  """
+        {
+          "top": {
+            "k": "8",
+            "k": "8",
+            "boom": 1
+          }
+        }
+        """
+        d = _pypyjson.loads(a)
+        str(d)
+        repr(d)
+

From pypy.commits at gmail.com  Thu Oct 31 16:14:24 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 31 Oct 2019 13:14:24 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix #3108: the map based parser didn't
 deal with json dicts with repeated keys
Message-ID: <5dbb40a0.1c69fb81.dc24d.bee2@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r97924:86da6cb357f1
Date: 2019-10-31 21:07 +0100
http://bitbucket.org/pypy/pypy/changeset/86da6cb357f1/

Log:	fix #3108: the map based parser didn't deal with json dicts with
	repeated keys correctly

diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -342,7 +342,14 @@
         currmap = self.startmap
         while True:
             # parse a key: value
-            currmap = self.decode_key_map(i, currmap)
+            newmap = self.decode_key_map(i, currmap)
+            if newmap is None:
+                # We've seen a repeated value, switch to dict-based storage.
+                dict_w = self._switch_to_dict(currmap, values_w, nextindex)
+                # We re-parse the last key, to get the correct overwriting
+                # effect. Pointless to care for performance here.
+                return self.decode_object_dict(i, start, dict_w)
+            currmap = newmap
             i = self.skip_whitespace(self.pos)
             ch = self.ll_chars[i]
             if ch != ':':
@@ -610,6 +617,8 @@
         """ Given the current map currmap of an object, decode the next key at
         position i. This returns the new map of the object. """
         newmap = self._decode_key_map(i, currmap)
+        if newmap is None:
+            return None
         currmap.observe_transition(newmap, self.startmap)
         return newmap
 
@@ -789,6 +798,11 @@
             self.nextmap_first._check_invariants()
 
     def get_next(self, w_key, string, start, stop, terminator):
+        """ Returns the next map, given a wrapped key w_key, the json input
+        string with positions start and stop, as well as a terminator.
+
+        Returns None if the key already appears somewhere in the map chain.
+        """
         from pypy.objspace.std.dictmultiobject import unicode_hash, unicode_eq
         if isinstance(self, JSONMap):
             assert not self.state == MapBase.BLOCKED
@@ -803,6 +817,8 @@
         if nextmap_first is None:
             # first transition ever seen, don't initialize nextmap_all
             next = self._make_next_map(w_key, string[start:stop])
+            if next is None:
+                return None
             self.nextmap_first = next
         else:
             if self.nextmap_all is None:
@@ -817,6 +833,8 @@
             # if we are at this point we didn't find the transition yet, so
             # create a new one
             next = self._make_next_map(w_key, string[start:stop])
+            if next is None:
+                return None
             self.nextmap_all[w_key] = next
 
             # one new leaf has been created
@@ -859,6 +877,14 @@
                 self.mark_useful(terminator)
 
     def _make_next_map(self, w_key, key_repr):
+        # Check whether w_key is already part of the self.prev chain
+        # to prevent strangeness in the json dict implementation.
+        # This is slow, but it should be rare to call this function.
+        check = self
+        while isinstance(check, JSONMap):
+            if check.w_key._utf8 == w_key._utf8:
+                return None
+            check = check.prev
         return JSONMap(self.space, self, w_key, key_repr)
 
     def fill_dict(self, dict_w, values_w):
diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py
--- a/pypy/module/_pypyjson/test/test__pypyjson.py
+++ b/pypy/module/_pypyjson/test/test__pypyjson.py
@@ -74,6 +74,17 @@
         m3.fill_dict(d, [space.w_None, space.w_None, space.w_None])
         assert list(d) == [w_a, w_b, w_c]
 
+    def test_repeated_key_get_next(self):
+        m = Terminator(self.space)
+        w_a = self.space.newutf8("a", 1)
+        w_b = self.space.newutf8("b", 1)
+        w_c = self.space.newutf8("c", 1)
+        m1 = m.get_next(w_a, '"a"', 0, 3, m)
+        m1 = m1.get_next(w_b, '"b"', 0, 3, m)
+        m1 = m1.get_next(w_c, '"c"', 0, 3, m)
+        m2 = m1.get_next(w_a, '"a"', 0, 3, m)
+        assert m2 is None
+
 
     def test_decode_key_map(self):
         m = Terminator(self.space)
@@ -519,3 +530,11 @@
             exc = raises(ValueError, _pypyjson.loads, inputtext)
             assert str(exc.value) == errmsg
 
+    def test_repeated_key(self):
+        import _pypyjson
+        a = '{"abc": "4", "k": 1, "k": 2}'
+        d = _pypyjson.loads(a)
+        assert d == {u"abc": u"4", u"k": 2}
+        a = '{"abc": "4", "k": 1, "k": 1.5, "c": null, "k": 2}'
+        d = _pypyjson.loads(a)
+        assert d == {u"abc": u"4", u"c": None, u"k": 2}

From pypy.commits at gmail.com  Thu Oct 31 23:02:52 2019
From: pypy.commits at gmail.com (mattip)
Date: Thu, 31 Oct 2019 20:02:52 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix failing test, remove tab
Message-ID: <5dbba05c.1c69fb81.c451.6696@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r97925:d91b02c62beb
Date: 2019-11-01 05:01 +0200
http://bitbucket.org/pypy/pypy/changeset/d91b02c62beb/

Log:	fix failing test, remove tab

diff --git a/lib_pypy/_curses_build.py b/lib_pypy/_curses_build.py
--- a/lib_pypy/_curses_build.py
+++ b/lib_pypy/_curses_build.py
@@ -21,7 +21,7 @@
     if os.path.exists('/usr/include/ncurses'):
         return ['/usr/include/ncurses']
     if os.path.exists('/usr/include/ncursesw'):
-    	return ['/usr/include/ncursesw']
+        return ['/usr/include/ncursesw']
     return []
 
 
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -1296,7 +1296,8 @@
         except TypeError as e:
             import sys
             if '__pypy__' in sys.builtin_module_names:
-                assert str(e) == 'instance layout conflicts in multiple inheritance'
+                print(str(e))
+                assert 'instance layout conflicts in multiple inheritance' in str(e)
 
             else:
                 assert str(e) == ('Error when calling the metaclass bases\n'