From pypy.commits at gmail.com  Sat May  4 09:55:56 2019
From: pypy.commits at gmail.com (arigo)
Date: Sat, 04 May 2019 06:55:56 -0700 (PDT)
Subject: [pypy-commit] pypy shadowstack-issue2722: Issue #2722: trying to
 make the size of the shadowstack more dynamic
Message-ID: <5ccd99ec.1c69fb81.fdd0f.710b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: shadowstack-issue2722
Changeset: r96569:e7b5c86ab3a3
Date: 2019-05-04 15:54 +0200
http://bitbucket.org/pypy/pypy/changeset/e7b5c86ab3a3/

Log:	Issue #2722: trying to make the size of the shadowstack more dynamic


From pypy.commits at gmail.com  Sat May  4 09:55:58 2019
From: pypy.commits at gmail.com (arigo)
Date: Sat, 04 May 2019 06:55:58 -0700 (PDT)
Subject: [pypy-commit] pypy shadowstack-issue2722: These changes might be
 enough to allow the shadowstack's address to
Message-ID: <5ccd99ee.1c69fb81.390c6.75a7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: shadowstack-issue2722
Changeset: r96570:c790dcca68e1
Date: 2019-05-04 15:55 +0200
http://bitbucket.org/pypy/pypy/changeset/c790dcca68e1/

Log:	These changes might be enough to allow the shadowstack's address to
	be changed at any point

diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py
--- a/rpython/translator/c/gc.py
+++ b/rpython/translator/c/gc.py
@@ -452,7 +452,6 @@
         #
         yield ('typedef struct { void %s; } pypy_ss_t;'
                    % ', '.join(['*s%d' % i for i in range(numcolors)]))
-        yield 'pypy_ss_t *ss;'
         funcgen.gcpol_ss = gcpol_ss
 
     def OP_GC_PUSH_ROOTS(self, funcgen, op):
@@ -462,26 +461,27 @@
         raise Exception("gc_pop_roots should be removed by postprocess_graph")
 
     def OP_GC_ENTER_ROOTS_FRAME(self, funcgen, op):
-        return 'ss = (pypy_ss_t *)%s; %s = (void *)(ss+1);' % (
-            funcgen.gcpol_ss, funcgen.gcpol_ss)
+        return '%s += sizeof(pypy_ss_t);' % (funcgen.gcpol_ss,)
 
     def OP_GC_LEAVE_ROOTS_FRAME(self, funcgen, op):
-        return '%s = (void *)ss;' % funcgen.gcpol_ss
+        return '%s -= sizeof(pypy_ss_t);' % (funcgen.gcpol_ss,)
 
     def OP_GC_SAVE_ROOT(self, funcgen, op):
         num = op.args[0].value
         exprvalue = funcgen.expr(op.args[1])
-        return 'ss->s%d = (void *)%s;\t/* gc_save_root */' % (num, exprvalue)
+        return '((pypy_ss_t *)%s)[-1].s%d = (void *)%s;' % (
+            funcgen.gcpol_ss, num, exprvalue)
 
     def OP_GC_RESTORE_ROOT(self, funcgen, op):
         num = op.args[0].value
         exprvalue = funcgen.expr(op.args[1])
         typename = funcgen.db.gettype(op.args[1].concretetype)
-        result = '%s = (%s)ss->s%d;' % (exprvalue, cdecl(typename, ''), num)
+        result = '%s = (%s)((pypy_ss_t *)%s)[-1].s%d;' % (
+            exprvalue, cdecl(typename, ''), funcgen.gcpol_ss, num)
         if isinstance(op.args[1], Constant):
-            return '/* %s\t* gc_restore_root */' % result
+            return '/* %s */' % result
         else:
-            return '%s\t/* gc_restore_root */' % result
+            return result
 
 
 class AsmGcRootFrameworkGcPolicy(BasicFrameworkGcPolicy):

From pypy.commits at gmail.com  Sat May  4 10:41:04 2019
From: pypy.commits at gmail.com (arigo)
Date: Sat, 04 May 2019 07:41:04 -0700 (PDT)
Subject: [pypy-commit] pypy shadowstack-issue2722: in-progress
Message-ID: <5ccda480.1c69fb81.a3672.ecfa@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: shadowstack-issue2722
Changeset: r96571:b47f28c00520
Date: 2019-05-04 16:40 +0200
http://bitbucket.org/pypy/pypy/changeset/b47f28c00520/

Log:	in-progress

diff --git a/rpython/memory/gctransform/shadowstack.py b/rpython/memory/gctransform/shadowstack.py
--- a/rpython/memory/gctransform/shadowstack.py
+++ b/rpython/memory/gctransform/shadowstack.py
@@ -95,7 +95,7 @@
         self.shadow_stack_pool = ShadowStackPool(gcdata)
         rsd = gctransformer.root_stack_depth
         if rsd is not None:
-            self.shadow_stack_pool.root_stack_depth = rsd
+            self.shadow_stack_pool.root_stack_size = sizeofaddr * rsd
 
     def push_stack(self, addr):
         top = self.incr_stack(1)
@@ -140,6 +140,7 @@
         # this is a dict {tid: SHADOWSTACKREF}, where the tid for the
         # current thread may be missing so far
         gcdata.thread_stacks = None
+        shadow_stack_pool.has_threads = True
 
         # Return the thread identifier, as an integer.
         get_tid = rthread.get_ident
@@ -268,7 +269,8 @@
     """Manages a pool of shadowstacks.
     """
     _alloc_flavor_ = "raw"
-    root_stack_depth = 163840
+    root_stack_size = sizeofaddr * 163840
+    has_threads = False
 
     def __init__(self, gcdata):
         self.unused_full_stack = llmemory.NULL
@@ -332,11 +334,47 @@
 
     def _prepare_unused_stack(self):
         if self.unused_full_stack == llmemory.NULL:
-            root_stack_size = sizeofaddr * self.root_stack_depth
-            self.unused_full_stack = llmemory.raw_malloc(root_stack_size)
+            self.unused_full_stack = llmemory.raw_malloc(self.root_stack_size)
             if self.unused_full_stack == llmemory.NULL:
                 raise MemoryError
 
+    def increase_root_stack_size(self, new_size):
+        if new_size <= self.root_stack_size:
+            return     # can't easily decrease the size
+        if self.unused_full_stack:
+            llmemory.raw_free(self.unused_full_stack)
+            self.unused_full_stack = llmemory.NULL
+        used = self.gcdata.root_stack_top - self.gcdata.root_stack_base
+        addr = self._resize(self.gcdata.root_stack_base, used, new_size)
+        self.gcdata.root_stack_base = addr
+        self.gcdata.root_stack_top  = addr + used
+        # no gc operations above: we just switched shadowstacks
+        if self.has_threads:
+            self._resize_thread_shadowstacks(new_size)
+        self.root_stack_size = new_size
+
+    def _resize_thread_shadowstacks(self, new_size):
+        if self.gcdata.thread_stacks is not None:
+            for ssref in self.gcdata.thread_stacks.values():
+                if ssref.base:
+                    used = ssref.top - ssref.base
+                    addr = self._resize(base, used, new_size)
+                    ssref.base = addr
+                    ssref.top = addr + used
+    _resize_thread_shadowstacks._dont_inline_ = True
+
+    def _resize(self, base, used, new_size):
+        ll_assert(used <= new_size, "shadowstack resize: overflow detected")
+        addr = llmemory.raw_malloc(new_size)
+        if addr == llmemory.NULL:
+            raise MemoryError
+        # note that we don't know the total memory size of 'base', but we
+        # know the size of the part that is used right now, and we only need
+        # to copy that
+        llmemory.raw_memmove(base, addr, used)
+        llmemory.raw_free(base)
+        return addr
+
 
 def get_shadowstackref(root_walker, gctransformer):
     if hasattr(gctransformer, '_SHADOWSTACKREF'):

From pypy.commits at gmail.com  Mon May  6 22:19:38 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 06 May 2019 19:19:38 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix for issue #3012,
 if no \r no need to translate
Message-ID: <5cd0eb3a.1c69fb81.10a41.e8dd@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r96572:436eebe7adb1
Date: 2019-05-06 21:52 -0400
http://bitbucket.org/pypy/pypy/changeset/436eebe7adb1/

Log:	fix for issue #3012, if no \r no need to translate

diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -105,16 +105,10 @@
         # desired, all in one pass.
         seennl = self.seennl
 
-        # If, up to now, newlines are consistently \n, do a quick check
-        # for the \r
-        only_lf = False
-        if seennl == SEEN_LF or seennl == 0:
-            only_lf = (output.find('\r') < 0)
-
-        if only_lf:
-            # If not already seen, quick scan for a possible "\n" character.
+        if output.find('\r') < 0:
+            # If no \r, quick scan for a possible "\n" character.
             # (there's nothing else to be done, even when in translation mode)
-            if seennl == 0 and output.find('\n') >= 0:
+            if output.find('\n') >= 0:
                 seennl |= SEEN_LF
                 # Finished: we have scanned for newlines, and none of them
                 # need translating.
diff --git a/pypy/module/_io/test/test_textio.py b/pypy/module/_io/test/test_textio.py
--- a/pypy/module/_io/test/test_textio.py
+++ b/pypy/module/_io/test/test_textio.py
@@ -377,3 +377,13 @@
         _check(dec)
         dec = _io.IncrementalNewlineDecoder(None, translate=True)
         _check(dec)
+
+    def test_newlines2(self):
+        import _io, codecs
+        inner_decoder = codecs.getincrementaldecoder("utf-8")()
+        decoder = _io.IncrementalNewlineDecoder(inner_decoder, translate=True)
+        msg = b"abc\r\n\n\r\r\n\n"
+        decoded = ''
+        for ch in msg:
+            decoded += decoder.decode(ch)
+        assert set(decoder.newlines) == {"\r", "\n", "\r\n"}

From pypy.commits at gmail.com  Mon May  6 22:19:40 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 06 May 2019 19:19:40 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: merge default into py3.6, fix test
Message-ID: <5cd0eb3c.1c69fb81.a3d62.18ae@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r96573:5420cbd1f137
Date: 2019-05-06 22:18 -0400
http://bitbucket.org/pypy/pypy/changeset/5420cbd1f137/

Log:	merge default into py3.6, fix test

diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -105,16 +105,10 @@
         # desired, all in one pass.
         seennl = self.seennl
 
-        # If, up to now, newlines are consistently \n, do a quick check
-        # for the \r
-        only_lf = False
-        if seennl == SEEN_LF or seennl == 0:
-            only_lf = (output.find('\r') < 0)
-
-        if only_lf:
-            # If not already seen, quick scan for a possible "\n" character.
+        if output.find('\r') < 0:
+            # If no \r, quick scan for a possible "\n" character.
             # (there's nothing else to be done, even when in translation mode)
-            if seennl == 0 and output.find('\n') >= 0:
+            if output.find('\n') >= 0:
                 seennl |= SEEN_LF
                 # Finished: we have scanned for newlines, and none of them
                 # need translating.
diff --git a/pypy/module/_io/test/test_textio.py b/pypy/module/_io/test/test_textio.py
--- a/pypy/module/_io/test/test_textio.py
+++ b/pypy/module/_io/test/test_textio.py
@@ -566,3 +566,13 @@
         _check(dec)
         dec = _io.IncrementalNewlineDecoder(None, translate=True)
         _check(dec)
+
+    def test_newlines2(self):
+        import _io, codecs
+        inner_decoder = codecs.getincrementaldecoder("utf-8")()
+        decoder = _io.IncrementalNewlineDecoder(inner_decoder, translate=True)
+        msg = b"abc\r\n\n\r\r\n\n"
+        decoded = ''
+        for ch in msg:
+            decoded += decoder.decode(bytes([ch]))
+        assert set(decoder.newlines) == {"\r", "\n", "\r\n"}

From pypy.commits at gmail.com  Wed May  8 06:28:34 2019
From: pypy.commits at gmail.com (arigo)
Date: Wed, 08 May 2019 03:28:34 -0700 (PDT)
Subject: [pypy-commit] pypy shadowstack-issue2722: Link
 'increase_root_stack_depth' through RPython and test it
Message-ID: <5cd2af52.1c69fb81.444aa.9e03@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: shadowstack-issue2722
Changeset: r96574:0f76d02c2314
Date: 2019-05-08 12:27 +0200
http://bitbucket.org/pypy/pypy/changeset/0f76d02c2314/

Log:	Link 'increase_root_stack_depth' through RPython and test it

diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -236,6 +236,9 @@
         if root_walker.stacklet_support:
             root_walker.need_stacklet_support(self, getfn)
 
+        if hasattr(root_walker, 'build_increase_root_stack_depth_ptr'):
+            root_walker.build_increase_root_stack_depth_ptr(getfn)
+
         self.layoutbuilder.encode_type_shapes_now()
         self.create_custom_trace_funcs(gcdata.gc, translator.rtyper)
 
@@ -1652,6 +1655,10 @@
         else:
             hop.rename("same_as")
 
+    def gct_gc_increase_root_stack_depth(self, hop):
+        hop.genop("direct_call",
+                  [self.root_walker.gc_increase_root_stack_depth_ptr,
+                   hop.spaceop.args[0]])
 
 
 class TransformerLayoutBuilder(gctypelayout.TypeLayoutBuilder):
diff --git a/rpython/memory/gctransform/shadowstack.py b/rpython/memory/gctransform/shadowstack.py
--- a/rpython/memory/gctransform/shadowstack.py
+++ b/rpython/memory/gctransform/shadowstack.py
@@ -95,7 +95,7 @@
         self.shadow_stack_pool = ShadowStackPool(gcdata)
         rsd = gctransformer.root_stack_depth
         if rsd is not None:
-            self.shadow_stack_pool.root_stack_size = sizeofaddr * rsd
+            self.shadow_stack_pool.root_stack_depth = rsd
 
     def push_stack(self, addr):
         top = self.incr_stack(1)
@@ -253,6 +253,15 @@
         self.gc_modified_shadowstack_ptr = getfn(gc_modified_shadowstack,
                                                  [], annmodel.s_None)
 
+    def build_increase_root_stack_depth_ptr(self, getfn):
+        shadow_stack_pool = self.shadow_stack_pool
+        def gc_increase_root_stack_depth(new_size):
+            shadow_stack_pool.increase_root_stack_depth(new_size)
+
+        self.gc_increase_root_stack_depth_ptr = getfn(
+                gc_increase_root_stack_depth, [annmodel.SomeInteger()],
+                annmodel.s_None)
+
     def postprocess_graph(self, gct, graph, any_inlining):
         from rpython.memory.gctransform import shadowcolor
         if any_inlining:
@@ -269,7 +278,7 @@
     """Manages a pool of shadowstacks.
     """
     _alloc_flavor_ = "raw"
-    root_stack_size = sizeofaddr * 163840
+    root_stack_depth = 163840
     has_threads = False
 
     def __init__(self, gcdata):
@@ -334,36 +343,38 @@
 
     def _prepare_unused_stack(self):
         if self.unused_full_stack == llmemory.NULL:
-            self.unused_full_stack = llmemory.raw_malloc(self.root_stack_size)
+            root_stack_size = sizeofaddr * self.root_stack_depth
+            self.unused_full_stack = llmemory.raw_malloc(root_stack_size)
             if self.unused_full_stack == llmemory.NULL:
                 raise MemoryError
 
-    def increase_root_stack_size(self, new_size):
-        if new_size <= self.root_stack_size:
+    def increase_root_stack_depth(self, new_depth):
+        if new_depth <= self.root_stack_depth:
             return     # can't easily decrease the size
         if self.unused_full_stack:
             llmemory.raw_free(self.unused_full_stack)
             self.unused_full_stack = llmemory.NULL
         used = self.gcdata.root_stack_top - self.gcdata.root_stack_base
-        addr = self._resize(self.gcdata.root_stack_base, used, new_size)
+        addr = self._resize(self.gcdata.root_stack_base, used, new_depth)
         self.gcdata.root_stack_base = addr
         self.gcdata.root_stack_top  = addr + used
         # no gc operations above: we just switched shadowstacks
         if self.has_threads:
-            self._resize_thread_shadowstacks(new_size)
-        self.root_stack_size = new_size
+            self._resize_thread_shadowstacks(new_depth)
+        self.root_stack_depth = new_depth
 
-    def _resize_thread_shadowstacks(self, new_size):
+    def _resize_thread_shadowstacks(self, new_depth):
         if self.gcdata.thread_stacks is not None:
             for ssref in self.gcdata.thread_stacks.values():
                 if ssref.base:
                     used = ssref.top - ssref.base
-                    addr = self._resize(base, used, new_size)
+                    addr = self._resize(base, used, new_depth)
                     ssref.base = addr
                     ssref.top = addr + used
     _resize_thread_shadowstacks._dont_inline_ = True
 
-    def _resize(self, base, used, new_size):
+    def _resize(self, base, used, new_depth):
+        new_size = sizeofaddr * new_depth
         ll_assert(used <= new_size, "shadowstack resize: overflow detected")
         addr = llmemory.raw_malloc(new_size)
         if addr == llmemory.NULL:
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -690,6 +690,13 @@
         hop.exception_cannot_occur()
         return hop.genop('gc_move_out_of_nursery', hop.args_v, resulttype=hop.r_result)
 
+ at jit.dont_look_inside
+def increase_root_stack_depth(new_depth):
+    """Shadowstack: make sure the size of the shadowstack is at least
+    'new_depth' pointers."""
+    from rpython.rtyper.lltypesystem.lloperation import llop
+    llop.gc_increase_root_stack_depth(lltype.Void, new_depth)
+
 # ____________________________________________________________
 
 
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -1226,6 +1226,9 @@
     def op_gc_move_out_of_nursery(self, obj):
         raise NotImplementedError("gc_move_out_of_nursery")
 
+    def op_gc_increase_root_stack_depth(self, new_depth):
+        raise NotImplementedError("gc_increase_root_stack_depth")
+
     def op_revdb_stop_point(self, *args):
         pass
     def op_revdb_send_answer(self, *args):
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -530,6 +530,7 @@
     'gc_rawrefcount_next_dead':         LLOp(),
 
     'gc_move_out_of_nursery':           LLOp(),
+    'gc_increase_root_stack_depth':     LLOp(),
 
     'gc_push_roots'        : LLOp(),  # temporary: list of roots to save
     'gc_pop_roots'         : LLOp(),  # temporary: list of roots to restore
diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py
--- a/rpython/rtyper/lltypesystem/opimpl.py
+++ b/rpython/rtyper/lltypesystem/opimpl.py
@@ -776,6 +776,9 @@
 def op_gc_move_out_of_nursery(obj):
     return obj
 
+def op_gc_increase_root_stack_depth(new_depth):
+    pass
+
 def op_revdb_do_next_call():
     pass
 
diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -1912,6 +1912,45 @@
     def test_total_gc_time(self):
         res = self.run("total_gc_time")
         assert res > 0 # should take a few microseconds
+
+    def define_increase_root_stack_depth(cls):
+        class X:
+            pass
+        def g(n):
+            if n <= 0:
+                return None
+            x = X()
+            x.n = n
+            x.next = g(n - 1)
+            return x
+        def f(depth):
+            from rpython.rlib.rstack import _stack_set_length_fraction
+            _stack_set_length_fraction(50.0)
+            # ^^^ the default is enough for at least 10'000 (but less than
+            # 100'000) recursions of the simple function g().  We multiply
+            # it by 50.0 to make sure that 200'000 works.  The default
+            # shadowstack depth is 163'840 entries, so 200'000 overflows
+            # that default shadowstack depth, and gives a segfault unless
+            # the following line works too.
+            from rpython.rlib.rgc import increase_root_stack_depth
+            increase_root_stack_depth(depth + 100)
+            #
+            g(depth)
+            return 42
+        return f
+
+    def test_increase_root_stack_depth(self):
+        if not sys.platform.startswith('linux'):
+            py.test.skip("linux only")
+        #
+        def myrunner(args):
+            args1 = ['/bin/bash', '-c', 'ulimit -s unlimited && %s' %
+                     (' '.join(args),)]
+            return subprocess.check_output(args1)
+        res = self.run("increase_root_stack_depth", 200000, runner=myrunner)
+        assert res == 42
+
+
 # ____________________________________________________________________
 
 class TaggedPointersTest(object):

From pypy.commits at gmail.com  Wed May  8 07:08:49 2019
From: pypy.commits at gmail.com (arigo)
Date: Wed, 08 May 2019 04:08:49 -0700 (PDT)
Subject: [pypy-commit] pypy shadowstack-issue2722: Use
 rgc.increase_root_stack_depth() from PyPy (+tweaks)
Message-ID: <5cd2b8c1.1c69fb81.9c91.cee7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: shadowstack-issue2722
Changeset: r96575:a203fe94e7b4
Date: 2019-05-08 12:44 +0200
http://bitbucket.org/pypy/pypy/changeset/a203fe94e7b4/

Log:	Use rgc.increase_root_stack_depth() from PyPy (+tweaks)

diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -51,10 +51,12 @@
 value to N reserves N/1000 times 768KB of stack space.
 """
     from rpython.rlib.rstack import _stack_set_length_fraction
+    from rpython.rlib.rgc import increase_root_stack_depth
     if new_limit <= 0:
         raise oefmt(space.w_ValueError, "recursion limit must be positive")
     space.sys.recursionlimit = new_limit
     _stack_set_length_fraction(new_limit * 0.001)
+    increase_root_stack_depth(int(new_limit * 0.001 * 163840))
 
 def getrecursionlimit(space):
     """Return the last value set by setrecursionlimit().
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -236,9 +236,6 @@
         if root_walker.stacklet_support:
             root_walker.need_stacklet_support(self, getfn)
 
-        if hasattr(root_walker, 'build_increase_root_stack_depth_ptr'):
-            root_walker.build_increase_root_stack_depth_ptr(getfn)
-
         self.layoutbuilder.encode_type_shapes_now()
         self.create_custom_trace_funcs(gcdata.gc, translator.rtyper)
 
@@ -575,6 +572,8 @@
             self.move_out_of_nursery_ptr = getfn(GCClass.move_out_of_nursery,
                                               [s_gc, SomeAddress()],
                                               SomeAddress())
+        if hasattr(self.root_walker, 'build_increase_root_stack_depth_ptr'):
+            self.root_walker.build_increase_root_stack_depth_ptr(getfn)
 
 
     def create_custom_trace_funcs(self, gc, rtyper):
@@ -1656,6 +1655,8 @@
             hop.rename("same_as")
 
     def gct_gc_increase_root_stack_depth(self, hop):
+        if not hasattr(self.root_walker, 'gc_increase_root_stack_depth_ptr'):
+            return
         hop.genop("direct_call",
                   [self.root_walker.gc_increase_root_stack_depth_ptr,
                    hop.spaceop.args[0]])
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -530,7 +530,7 @@
     'gc_rawrefcount_next_dead':         LLOp(),
 
     'gc_move_out_of_nursery':           LLOp(),
-    'gc_increase_root_stack_depth':     LLOp(),
+    'gc_increase_root_stack_depth':     LLOp(canrun=True),
 
     'gc_push_roots'        : LLOp(),  # temporary: list of roots to save
     'gc_pop_roots'         : LLOp(),  # temporary: list of roots to restore

From pypy.commits at gmail.com  Wed May  8 08:33:19 2019
From: pypy.commits at gmail.com (arigo)
Date: Wed, 08 May 2019 05:33:19 -0700 (PDT)
Subject: [pypy-commit] pypy shadowstack-issue2722: fix
Message-ID: <5cd2cc8f.1c69fb81.f9e57.cc47@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: shadowstack-issue2722
Changeset: r96576:82825fc607bb
Date: 2019-05-08 14:32 +0200
http://bitbucket.org/pypy/pypy/changeset/82825fc607bb/

Log:	fix

diff --git a/rpython/memory/gctransform/shadowstack.py b/rpython/memory/gctransform/shadowstack.py
--- a/rpython/memory/gctransform/shadowstack.py
+++ b/rpython/memory/gctransform/shadowstack.py
@@ -368,7 +368,7 @@
             for ssref in self.gcdata.thread_stacks.values():
                 if ssref.base:
                     used = ssref.top - ssref.base
-                    addr = self._resize(base, used, new_depth)
+                    addr = self._resize(ssref.base, used, new_depth)
                     ssref.base = addr
                     ssref.top = addr + used
     _resize_thread_shadowstacks._dont_inline_ = True

From pypy.commits at gmail.com  Wed May  8 09:23:42 2019
From: pypy.commits at gmail.com (arigo)
Date: Wed, 08 May 2019 06:23:42 -0700 (PDT)
Subject: [pypy-commit] pypy shadowstack-issue2722: Seems to work as expected,
 writing down a comment here after I debugged
Message-ID: <5cd2d85e.1c69fb81.a1809.60f4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: shadowstack-issue2722
Changeset: r96577:d92c072bcab5
Date: 2019-05-08 15:22 +0200
http://bitbucket.org/pypy/pypy/changeset/d92c072bcab5/

Log:	Seems to work as expected, writing down a comment here after I
	debugged a multi-threaded problem that doesn't exist

diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -49,6 +49,11 @@
 reserves 768KB of stack space, which should suffice (on Linux,
 depending on the compiler settings) for ~1400 calls.  Setting the
 value to N reserves N/1000 times 768KB of stack space.
+
+Note that there are other factors that also limit the stack size.
+The operating system typically sets a maximum which can be changed
+manually (e.g. with "ulimit" on Linux) for the main thread.  For other
+threads you can configure the limit by calling "threading.stack_size()".
 """
     from rpython.rlib.rstack import _stack_set_length_fraction
     from rpython.rlib.rgc import increase_root_stack_depth

From pypy.commits at gmail.com  Wed May  8 09:24:12 2019
From: pypy.commits at gmail.com (arigo)
Date: Wed, 08 May 2019 06:24:12 -0700 (PDT)
Subject: [pypy-commit] pypy shadowstack-issue2722: close branch,
 ready to merge
Message-ID: <5cd2d87c.1c69fb81.a4567.b937@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: shadowstack-issue2722
Changeset: r96578:d14d5e009ca0
Date: 2019-05-08 15:23 +0200
http://bitbucket.org/pypy/pypy/changeset/d14d5e009ca0/

Log:	close branch, ready to merge


From pypy.commits at gmail.com  Wed May  8 09:24:14 2019
From: pypy.commits at gmail.com (arigo)
Date: Wed, 08 May 2019 06:24:14 -0700 (PDT)
Subject: [pypy-commit] pypy default: hg merge shadowstack-issue2722
Message-ID: <5cd2d87e.1c69fb81.444aa.e0f6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r96579:52462bc541a3
Date: 2019-05-08 15:23 +0200
http://bitbucket.org/pypy/pypy/changeset/52462bc541a3/

Log:	hg merge shadowstack-issue2722

diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -49,12 +49,19 @@
 reserves 768KB of stack space, which should suffice (on Linux,
 depending on the compiler settings) for ~1400 calls.  Setting the
 value to N reserves N/1000 times 768KB of stack space.
+
+Note that there are other factors that also limit the stack size.
+The operating system typically sets a maximum which can be changed
+manually (e.g. with "ulimit" on Linux) for the main thread.  For other
+threads you can configure the limit by calling "threading.stack_size()".
 """
     from rpython.rlib.rstack import _stack_set_length_fraction
+    from rpython.rlib.rgc import increase_root_stack_depth
     if new_limit <= 0:
         raise oefmt(space.w_ValueError, "recursion limit must be positive")
     space.sys.recursionlimit = new_limit
     _stack_set_length_fraction(new_limit * 0.001)
+    increase_root_stack_depth(int(new_limit * 0.001 * 163840))
 
 def getrecursionlimit(space):
     """Return the last value set by setrecursionlimit().
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -572,6 +572,8 @@
             self.move_out_of_nursery_ptr = getfn(GCClass.move_out_of_nursery,
                                               [s_gc, SomeAddress()],
                                               SomeAddress())
+        if hasattr(self.root_walker, 'build_increase_root_stack_depth_ptr'):
+            self.root_walker.build_increase_root_stack_depth_ptr(getfn)
 
 
     def create_custom_trace_funcs(self, gc, rtyper):
@@ -1652,6 +1654,12 @@
         else:
             hop.rename("same_as")
 
+    def gct_gc_increase_root_stack_depth(self, hop):
+        if not hasattr(self.root_walker, 'gc_increase_root_stack_depth_ptr'):
+            return
+        hop.genop("direct_call",
+                  [self.root_walker.gc_increase_root_stack_depth_ptr,
+                   hop.spaceop.args[0]])
 
 
 class TransformerLayoutBuilder(gctypelayout.TypeLayoutBuilder):
diff --git a/rpython/memory/gctransform/shadowstack.py b/rpython/memory/gctransform/shadowstack.py
--- a/rpython/memory/gctransform/shadowstack.py
+++ b/rpython/memory/gctransform/shadowstack.py
@@ -140,6 +140,7 @@
         # this is a dict {tid: SHADOWSTACKREF}, where the tid for the
         # current thread may be missing so far
         gcdata.thread_stacks = None
+        shadow_stack_pool.has_threads = True
 
         # Return the thread identifier, as an integer.
         get_tid = rthread.get_ident
@@ -252,6 +253,15 @@
         self.gc_modified_shadowstack_ptr = getfn(gc_modified_shadowstack,
                                                  [], annmodel.s_None)
 
+    def build_increase_root_stack_depth_ptr(self, getfn):
+        shadow_stack_pool = self.shadow_stack_pool
+        def gc_increase_root_stack_depth(new_size):
+            shadow_stack_pool.increase_root_stack_depth(new_size)
+
+        self.gc_increase_root_stack_depth_ptr = getfn(
+                gc_increase_root_stack_depth, [annmodel.SomeInteger()],
+                annmodel.s_None)
+
     def postprocess_graph(self, gct, graph, any_inlining):
         from rpython.memory.gctransform import shadowcolor
         if any_inlining:
@@ -269,6 +279,7 @@
     """
     _alloc_flavor_ = "raw"
     root_stack_depth = 163840
+    has_threads = False
 
     def __init__(self, gcdata):
         self.unused_full_stack = llmemory.NULL
@@ -337,6 +348,44 @@
             if self.unused_full_stack == llmemory.NULL:
                 raise MemoryError
 
+    def increase_root_stack_depth(self, new_depth):
+        if new_depth <= self.root_stack_depth:
+            return     # can't easily decrease the size
+        if self.unused_full_stack:
+            llmemory.raw_free(self.unused_full_stack)
+            self.unused_full_stack = llmemory.NULL
+        used = self.gcdata.root_stack_top - self.gcdata.root_stack_base
+        addr = self._resize(self.gcdata.root_stack_base, used, new_depth)
+        self.gcdata.root_stack_base = addr
+        self.gcdata.root_stack_top  = addr + used
+        # no gc operations above: we just switched shadowstacks
+        if self.has_threads:
+            self._resize_thread_shadowstacks(new_depth)
+        self.root_stack_depth = new_depth
+
+    def _resize_thread_shadowstacks(self, new_depth):
+        if self.gcdata.thread_stacks is not None:
+            for ssref in self.gcdata.thread_stacks.values():
+                if ssref.base:
+                    used = ssref.top - ssref.base
+                    addr = self._resize(ssref.base, used, new_depth)
+                    ssref.base = addr
+                    ssref.top = addr + used
+    _resize_thread_shadowstacks._dont_inline_ = True
+
+    def _resize(self, base, used, new_depth):
+        new_size = sizeofaddr * new_depth
+        ll_assert(used <= new_size, "shadowstack resize: overflow detected")
+        addr = llmemory.raw_malloc(new_size)
+        if addr == llmemory.NULL:
+            raise MemoryError
+        # note that we don't know the total memory size of 'base', but we
+        # know the size of the part that is used right now, and we only need
+        # to copy that
+        llmemory.raw_memmove(base, addr, used)
+        llmemory.raw_free(base)
+        return addr
+
 
 def get_shadowstackref(root_walker, gctransformer):
     if hasattr(gctransformer, '_SHADOWSTACKREF'):
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -690,6 +690,13 @@
         hop.exception_cannot_occur()
         return hop.genop('gc_move_out_of_nursery', hop.args_v, resulttype=hop.r_result)
 
+ at jit.dont_look_inside
+def increase_root_stack_depth(new_depth):
+    """Shadowstack: make sure the size of the shadowstack is at least
+    'new_depth' pointers."""
+    from rpython.rtyper.lltypesystem.lloperation import llop
+    llop.gc_increase_root_stack_depth(lltype.Void, new_depth)
+
 # ____________________________________________________________
 
 
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -1226,6 +1226,9 @@
     def op_gc_move_out_of_nursery(self, obj):
         raise NotImplementedError("gc_move_out_of_nursery")
 
+    def op_gc_increase_root_stack_depth(self, new_depth):
+        raise NotImplementedError("gc_increase_root_stack_depth")
+
     def op_revdb_stop_point(self, *args):
         pass
     def op_revdb_send_answer(self, *args):
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -530,6 +530,7 @@
     'gc_rawrefcount_next_dead':         LLOp(),
 
     'gc_move_out_of_nursery':           LLOp(),
+    'gc_increase_root_stack_depth':     LLOp(canrun=True),
 
     'gc_push_roots'        : LLOp(),  # temporary: list of roots to save
     'gc_pop_roots'         : LLOp(),  # temporary: list of roots to restore
diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py
--- a/rpython/rtyper/lltypesystem/opimpl.py
+++ b/rpython/rtyper/lltypesystem/opimpl.py
@@ -776,6 +776,9 @@
 def op_gc_move_out_of_nursery(obj):
     return obj
 
+def op_gc_increase_root_stack_depth(new_depth):
+    pass
+
 def op_revdb_do_next_call():
     pass
 
diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py
--- a/rpython/translator/c/gc.py
+++ b/rpython/translator/c/gc.py
@@ -452,7 +452,6 @@
         #
         yield ('typedef struct { void %s; } pypy_ss_t;'
                    % ', '.join(['*s%d' % i for i in range(numcolors)]))
-        yield 'pypy_ss_t *ss;'
         funcgen.gcpol_ss = gcpol_ss
 
     def OP_GC_PUSH_ROOTS(self, funcgen, op):
@@ -462,26 +461,27 @@
         raise Exception("gc_pop_roots should be removed by postprocess_graph")
 
     def OP_GC_ENTER_ROOTS_FRAME(self, funcgen, op):
-        return 'ss = (pypy_ss_t *)%s; %s = (void *)(ss+1);' % (
-            funcgen.gcpol_ss, funcgen.gcpol_ss)
+        return '%s += sizeof(pypy_ss_t);' % (funcgen.gcpol_ss,)
 
     def OP_GC_LEAVE_ROOTS_FRAME(self, funcgen, op):
-        return '%s = (void *)ss;' % funcgen.gcpol_ss
+        return '%s -= sizeof(pypy_ss_t);' % (funcgen.gcpol_ss,)
 
     def OP_GC_SAVE_ROOT(self, funcgen, op):
         num = op.args[0].value
         exprvalue = funcgen.expr(op.args[1])
-        return 'ss->s%d = (void *)%s;\t/* gc_save_root */' % (num, exprvalue)
+        return '((pypy_ss_t *)%s)[-1].s%d = (void *)%s;' % (
+            funcgen.gcpol_ss, num, exprvalue)
 
     def OP_GC_RESTORE_ROOT(self, funcgen, op):
         num = op.args[0].value
         exprvalue = funcgen.expr(op.args[1])
         typename = funcgen.db.gettype(op.args[1].concretetype)
-        result = '%s = (%s)ss->s%d;' % (exprvalue, cdecl(typename, ''), num)
+        result = '%s = (%s)((pypy_ss_t *)%s)[-1].s%d;' % (
+            exprvalue, cdecl(typename, ''), funcgen.gcpol_ss, num)
         if isinstance(op.args[1], Constant):
-            return '/* %s\t* gc_restore_root */' % result
+            return '/* %s */' % result
         else:
-            return '%s\t/* gc_restore_root */' % result
+            return result
 
 
 class AsmGcRootFrameworkGcPolicy(BasicFrameworkGcPolicy):
diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -1912,6 +1912,45 @@
     def test_total_gc_time(self):
         res = self.run("total_gc_time")
         assert res > 0 # should take a few microseconds
+
+    def define_increase_root_stack_depth(cls):
+        class X:
+            pass
+        def g(n):
+            if n <= 0:
+                return None
+            x = X()
+            x.n = n
+            x.next = g(n - 1)
+            return x
+        def f(depth):
+            from rpython.rlib.rstack import _stack_set_length_fraction
+            _stack_set_length_fraction(50.0)
+            # ^^^ the default is enough for at least 10'000 (but less than
+            # 100'000) recursions of the simple function g().  We multiply
+            # it by 50.0 to make sure that 200'000 works.  The default
+            # shadowstack depth is 163'840 entries, so 200'000 overflows
+            # that default shadowstack depth, and gives a segfault unless
+            # the following line works too.
+            from rpython.rlib.rgc import increase_root_stack_depth
+            increase_root_stack_depth(depth + 100)
+            #
+            g(depth)
+            return 42
+        return f
+
+    def test_increase_root_stack_depth(self):
+        if not sys.platform.startswith('linux'):
+            py.test.skip("linux only")
+        #
+        def myrunner(args):
+            args1 = ['/bin/bash', '-c', 'ulimit -s unlimited && %s' %
+                     (' '.join(args),)]
+            return subprocess.check_output(args1)
+        res = self.run("increase_root_stack_depth", 200000, runner=myrunner)
+        assert res == 42
+
+
 # ____________________________________________________________________
 
 class TaggedPointersTest(object):

From pypy.commits at gmail.com  Thu May  9 10:39:57 2019
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 09 May 2019 07:39:57 -0700 (PDT)
Subject: [pypy-commit] pypy optimizeopt-cleanup: move some use_unroll logic
 out of compile_loop()
Message-ID: <5cd43bbd.1c69fb81.c6c29.5cd7@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: optimizeopt-cleanup
Changeset: r96580:37157d69c7e4
Date: 2019-04-27 17:00 +0100
http://bitbucket.org/pypy/pypy/changeset/37157d69c7e4/

Log:	move some use_unroll logic out of compile_loop()

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -2600,15 +2600,17 @@
                     self.staticdata.log('cancelled too many times!')
                     raise SwitchToBlackhole(Counters.ABORT_BAD_LOOP)
             else:
+                can_use_unroll = (self.staticdata.cpu.supports_guard_gc_type and
+                    'unroll' in self.jitdriver_sd.warmstate.enable_opts)
                 target_token = self.compile_loop(
-                    original_boxes, live_arg_boxes, start)
+                    original_boxes, live_arg_boxes, start, can_use_unroll=can_use_unroll)
                 self.raise_if_successful(live_arg_boxes, target_token)
                 # creation of the loop was cancelled!
                 self.cancel_count += 1
                 if self.cancelled_too_many_times():
                     target_token = self.compile_loop(
                         original_boxes, live_arg_boxes, start,
-                        try_disabling_unroll=True)
+                        try_disabling_unroll=True, can_use_unroll=can_use_unroll)
                     self.raise_if_successful(live_arg_boxes, target_token)
                     #
                     self.staticdata.log('cancelled too many times!')
@@ -2733,7 +2735,12 @@
         return cell.get_procedure_token()
 
     def compile_loop(self, original_boxes, live_arg_boxes, start,
-                     try_disabling_unroll=False):
+                     try_disabling_unroll=False, can_use_unroll=True):
+        use_unroll = can_use_unroll
+        if try_disabling_unroll:
+            if not use_unroll:
+                return
+            use_unroll = False
         num_green_args = self.jitdriver_sd.num_green_args
         greenkey = original_boxes[:num_green_args]
         ptoken = self.get_procedure_token(greenkey)
@@ -2741,12 +2748,6 @@
             # XXX this path not tested, but shown to occur on pypy-c :-(
             self.staticdata.log('cancelled: we already have a token now')
             raise SwitchToBlackhole(Counters.ABORT_BAD_LOOP)
-        use_unroll = (self.staticdata.cpu.supports_guard_gc_type and
-            'unroll' in self.jitdriver_sd.warmstate.enable_opts)
-        if try_disabling_unroll:
-            if not use_unroll:
-                return
-            use_unroll = False
         target_token = compile.compile_loop(
             self, greenkey, start, original_boxes[num_green_args:],
             live_arg_boxes[num_green_args:], use_unroll=use_unroll)
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -2885,7 +2885,7 @@
             return i
         #
         def my_compile_loop(self, original_boxes, live_arg_boxes, start,
-                        try_disabling_unroll=False, exported_state=None):
+                        try_disabling_unroll=False, can_use_unroll=None):
             return None
         old_compile_loop = MetaInterp.compile_loop
         MetaInterp.compile_loop = my_compile_loop
@@ -2919,7 +2919,7 @@
         #
         seen = []
         def my_compile_loop(self, original_boxes, live_arg_boxes, start,
-                        try_disabling_unroll=False, exported_state=None):
+                        try_disabling_unroll=False, can_use_unroll=None):
             seen.append(try_disabling_unroll)
             return None
         old_compile_loop = MetaInterp.compile_loop

From pypy.commits at gmail.com  Thu May  9 10:40:00 2019
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 09 May 2019 07:40:00 -0700 (PDT)
Subject: [pypy-commit] pypy optimizeopt-cleanup: Move use_unroll logic
 completely out of compile_loop()
Message-ID: <5cd43bc0.1c69fb81.83b53.5379@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: optimizeopt-cleanup
Changeset: r96581:11de4273c126
Date: 2019-04-29 19:52 +0100
http://bitbucket.org/pypy/pypy/changeset/11de4273c126/

Log:	Move use_unroll logic completely out of compile_loop()

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -2579,6 +2579,8 @@
         # green keys, representing the beginning of the same loop as the one
         # we end now.
 
+        can_use_unroll = (self.staticdata.cpu.supports_guard_gc_type and
+            'unroll' in self.jitdriver_sd.warmstate.enable_opts)
         for j in range(len(self.current_merge_points)-1, -1, -1):
             original_boxes, start = self.current_merge_points[j]
             assert len(original_boxes) == len(live_arg_boxes)
@@ -2600,18 +2602,19 @@
                     self.staticdata.log('cancelled too many times!')
                     raise SwitchToBlackhole(Counters.ABORT_BAD_LOOP)
             else:
-                can_use_unroll = (self.staticdata.cpu.supports_guard_gc_type and
-                    'unroll' in self.jitdriver_sd.warmstate.enable_opts)
                 target_token = self.compile_loop(
-                    original_boxes, live_arg_boxes, start, can_use_unroll=can_use_unroll)
+                    original_boxes, live_arg_boxes, start,
+                    use_unroll=can_use_unroll)
                 self.raise_if_successful(live_arg_boxes, target_token)
                 # creation of the loop was cancelled!
                 self.cancel_count += 1
                 if self.cancelled_too_many_times():
-                    target_token = self.compile_loop(
-                        original_boxes, live_arg_boxes, start,
-                        try_disabling_unroll=True, can_use_unroll=can_use_unroll)
-                    self.raise_if_successful(live_arg_boxes, target_token)
+                    if can_use_unroll:
+                        # try one last time without unrolling
+                        target_token = self.compile_loop(
+                            original_boxes, live_arg_boxes, start,
+                            use_unroll=False)
+                        self.raise_if_successful(live_arg_boxes, target_token)
                     #
                     self.staticdata.log('cancelled too many times!')
                     raise SwitchToBlackhole(Counters.ABORT_BAD_LOOP)
@@ -2734,13 +2737,7 @@
             return None
         return cell.get_procedure_token()
 
-    def compile_loop(self, original_boxes, live_arg_boxes, start,
-                     try_disabling_unroll=False, can_use_unroll=True):
-        use_unroll = can_use_unroll
-        if try_disabling_unroll:
-            if not use_unroll:
-                return
-            use_unroll = False
+    def compile_loop(self, original_boxes, live_arg_boxes, start, use_unroll):
         num_green_args = self.jitdriver_sd.num_green_args
         greenkey = original_boxes[:num_green_args]
         ptoken = self.get_procedure_token(greenkey)
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -2884,8 +2884,8 @@
                 i += 1
             return i
         #
-        def my_compile_loop(self, original_boxes, live_arg_boxes, start,
-                        try_disabling_unroll=False, can_use_unroll=None):
+        def my_compile_loop(
+                self, original_boxes, live_arg_boxes, start, use_unroll):
             return None
         old_compile_loop = MetaInterp.compile_loop
         MetaInterp.compile_loop = my_compile_loop
@@ -2918,9 +2918,9 @@
             return i
         #
         seen = []
-        def my_compile_loop(self, original_boxes, live_arg_boxes, start,
-                        try_disabling_unroll=False, can_use_unroll=None):
-            seen.append(try_disabling_unroll)
+        def my_compile_loop(
+                self, original_boxes, live_arg_boxes, start, use_unroll):
+            seen.append(use_unroll)
             return None
         old_compile_loop = MetaInterp.compile_loop
         MetaInterp.compile_loop = my_compile_loop

From pypy.commits at gmail.com  Thu May  9 10:40:01 2019
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 09 May 2019 07:40:01 -0700 (PDT)
Subject: [pypy-commit] pypy optimizeopt-cleanup: Move
 record_loop_or_bridge() call inside compile_and_attach()
Message-ID: <5cd43bc1.1c69fb81.16d1d.ec1e@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: optimizeopt-cleanup
Changeset: r96582:db5154f11410
Date: 2019-05-02 04:23 +0100
http://bitbucket.org/pypy/pypy/changeset/db5154f11410/

Log:	Move record_loop_or_bridge() call inside compile_and_attach()

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -164,19 +164,14 @@
     original_jitcell_token = loop.original_jitcell_token
     assert original_jitcell_token is not None
     if metainterp_sd.warmrunnerdesc is not None:    # for tests
-        assert original_jitcell_token.generation > 0     # has been registered with memmgr
+        assert original_jitcell_token.generation > 0  # has been registered with memmgr
     wref = weakref.ref(original_jitcell_token)
     clt = original_jitcell_token.compiled_loop_token
     clt.loop_token_wref = wref
     for op in loop.operations:
         descr = op.getdescr()
-        # not sure what descr.index is about
         if isinstance(descr, ResumeDescr):
             descr.rd_loop_token = clt   # stick it there
-            #n = descr.index
-            #if n >= 0:       # we also record the resumedescr number
-            #    original_jitcell_token.compiled_loop_token.record_faildescr_index(n)
-        #    pass
         if isinstance(descr, JitCellToken):
             # for a CALL_ASSEMBLER: record it as a potential jump.
             if descr is not original_jitcell_token:
@@ -401,8 +396,6 @@
 
     target_token = loop.operations[-1].getdescr()
     resumekey.compile_and_attach(metainterp, loop, inputargs)
-
-    record_loop_or_bridge(metainterp_sd, loop)
     return target_token
 
 def get_box_replacement(op, allow_none=False):
@@ -810,6 +803,7 @@
                                self, inputargs, new_loop.operations,
                                new_loop.original_jitcell_token,
                                metainterp.box_names_memo)
+        record_loop_or_bridge(metainterp.staticdata, new_loop)
 
     def make_a_counter_per_value(self, guard_value_op, index):
         assert guard_value_op.getopnum() == rop.GUARD_VALUE
@@ -1020,6 +1014,7 @@
         jitdriver_sd.warmstate.attach_procedure_to_interp(
             self.original_greenkey, jitcell_token)
         metainterp_sd.stats.add_jitcell_token(jitcell_token)
+        record_loop_or_bridge(metainterp_sd, new_loop)
 
     def get_resumestorage(self):
         return None
@@ -1084,7 +1079,6 @@
         new_trace.inputargs = info.inputargs
         target_token = new_trace.operations[-1].getdescr()
         resumekey.compile_and_attach(metainterp, new_trace, inputargs)
-        record_loop_or_bridge(metainterp_sd, new_trace)
         return target_token
     new_trace.inputargs = info.renamed_inputargs
     metainterp.retrace_needed(new_trace, info)
diff --git a/rpython/jit/metainterp/optimizeopt/__init__.py b/rpython/jit/metainterp/optimizeopt/__init__.py
--- a/rpython/jit/metainterp/optimizeopt/__init__.py
+++ b/rpython/jit/metainterp/optimizeopt/__init__.py
@@ -53,12 +53,12 @@
 def optimize_trace(metainterp_sd, jitdriver_sd, compile_data, memo=None):
     """Optimize loop.operations to remove internal overheadish operations.
     """
+    if memo is None:
+        memo = {}
     debug_start("jit-optimize")
     try:
         _log_loop_from_trace(metainterp_sd, compile_data.trace, memo,
                              is_unrolled=not compile_data.log_noopt)
-        if memo is None:
-            memo = {}
         compile_data.box_names_memo = memo
         optimizations = build_opt_chain(compile_data.enable_opts)
         return compile_data.optimize(

From pypy.commits at gmail.com  Thu May  9 10:40:03 2019
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 09 May 2019 07:40:03 -0700 (PDT)
Subject: [pypy-commit] pypy optimizeopt-cleanup: cleanup
Message-ID: <5cd43bc3.1c69fb81.9ccab.1456@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: optimizeopt-cleanup
Changeset: r96583:40e26f9f4337
Date: 2019-05-07 19:48 +0100
http://bitbucket.org/pypy/pypy/changeset/40e26f9f4337/

Log:	cleanup

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -1024,7 +1024,6 @@
     """Try to compile a new bridge leading from the beginning of the history
     to some existing place.
     """
-
     from rpython.jit.metainterp.optimizeopt import optimize_trace
 
     # The history contains new operations to attach as the code for the
@@ -1035,11 +1034,6 @@
 
     metainterp_sd = metainterp.staticdata
     jitdriver_sd = metainterp.jitdriver_sd
-    #
-    jd_name = jitdriver_sd.jitdriver.name
-    metainterp_sd.jitlog.start_new_trace(metainterp_sd,
-            faildescr=resumekey, entry_bridge=False, jd_name=jd_name)
-    #
     if isinstance(resumekey, ResumeAtPositionDescr):
         inline_short_preamble = False
     else:
@@ -1048,10 +1042,14 @@
     trace = metainterp.history.trace
     jitdriver_sd = metainterp.jitdriver_sd
     enable_opts = jitdriver_sd.warmstate.enable_opts
-
     call_pure_results = metainterp.call_pure_results
     resumestorage = resumekey.get_resumestorage()
 
+    trace.tracing_done()
+    metainterp_sd.jitlog.start_new_trace(metainterp_sd,
+        faildescr=resumekey, entry_bridge=False,
+        jd_name=jitdriver_sd.jitdriver.name)
+
     if metainterp.history.ends_with_jump:
         data = BridgeCompileData(trace, runtime_boxes, resumestorage,
                                  call_pure_results=call_pure_results,
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -2771,7 +2771,6 @@
         self.history.record(rop.JUMP, live_arg_boxes[num_green_args:], None,
                             descr=ptoken)
         self.history.ends_with_jump = True
-        self.history.trace.tracing_done()
         try:
             target_token = compile.compile_trace(self, self.resumekey,
                 live_arg_boxes[num_green_args:])
@@ -2781,7 +2780,6 @@
         self.raise_if_successful(live_arg_boxes, target_token)
 
     def compile_done_with_this_frame(self, exitbox):
-        # temporarily put a JUMP to a pseudo-loop
         self.store_token_in_vable()
         sd = self.staticdata
         result_type = self.jitdriver_sd.result_type
@@ -2800,9 +2798,7 @@
             token = sd.done_with_this_frame_descr_float
         else:
             assert False
-        # FIXME: can we call compile_trace?
         self.history.record(rop.FINISH, exits, None, descr=token)
-        self.history.trace.tracing_done()
         target_token = compile.compile_trace(self, self.resumekey, exits)
         if target_token is not token:
             compile.giveup()
@@ -2828,7 +2824,6 @@
         sd = self.staticdata
         token = sd.exit_frame_with_exception_descr_ref
         self.history.record(rop.FINISH, [valuebox], None, descr=token)
-        self.history.trace.tracing_done()
         target_token = compile.compile_trace(self, self.resumekey, [valuebox])
         if target_token is not token:
             compile.giveup()

From pypy.commits at gmail.com  Thu May  9 10:40:05 2019
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 09 May 2019 07:40:05 -0700 (PDT)
Subject: [pypy-commit] pypy optimizeopt-cleanup: Turn History.ends_with_jump
 into a parameter for compile_trace()
Message-ID: <5cd43bc5.1c69fb81.e650f.1d82@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: optimizeopt-cleanup
Changeset: r96584:ed1d19665ec6
Date: 2019-05-08 02:35 +0100
http://bitbucket.org/pypy/pypy/changeset/ed1d19665ec6/

Log:	Turn History.ends_with_jump into a parameter for compile_trace()

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -1020,7 +1020,7 @@
         return None
 
 
-def compile_trace(metainterp, resumekey, runtime_boxes):
+def compile_trace(metainterp, resumekey, runtime_boxes, ends_with_jump=False):
     """Try to compile a new bridge leading from the beginning of the history
     to some existing place.
     """
@@ -1050,7 +1050,7 @@
         faildescr=resumekey, entry_bridge=False,
         jd_name=jitdriver_sd.jitdriver.name)
 
-    if metainterp.history.ends_with_jump:
+    if ends_with_jump:
         data = BridgeCompileData(trace, runtime_boxes, resumestorage,
                                  call_pure_results=call_pure_results,
                                  enable_opts=enable_opts,
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -689,7 +689,6 @@
 
 
 class History(object):
-    ends_with_jump = False
     trace = None
 
     def __init__(self):
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -2770,13 +2770,11 @@
         self.potential_retrace_position = cut_at
         self.history.record(rop.JUMP, live_arg_boxes[num_green_args:], None,
                             descr=ptoken)
-        self.history.ends_with_jump = True
         try:
             target_token = compile.compile_trace(self, self.resumekey,
-                live_arg_boxes[num_green_args:])
+                live_arg_boxes[num_green_args:], ends_with_jump=True)
         finally:
             self.history.cut(cut_at)  # pop the jump
-            self.history.ends_with_jump = False
         self.raise_if_successful(live_arg_boxes, target_token)
 
     def compile_done_with_this_frame(self, exitbox):

From pypy.commits at gmail.com  Thu May  9 10:40:07 2019
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 09 May 2019 07:40:07 -0700 (PDT)
Subject: [pypy-commit] pypy optimizeopt-cleanup: Make optimize_trace() a
 method of CompileData
Message-ID: <5cd43bc7.1c69fb81.bdd4f.7239@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: optimizeopt-cleanup
Changeset: r96585:2853f7476ba2
Date: 2019-05-09 15:38 +0100
http://bitbucket.org/pypy/pypy/changeset/2853f7476ba2/

Log:	Make optimize_trace() a method of CompileData

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -39,6 +39,26 @@
         for arg in self.trace.inputargs:
             arg.set_forwarded(None)
 
+    def optimize_trace(self, metainterp_sd, jitdriver_sd, memo):
+        """Optimize loop.operations to remove internal overheadish operations.
+        """
+        from rpython.jit.metainterp.optimizeopt import build_opt_chain
+        # mark that a new trace has been started
+        log = metainterp_sd.jitlog.log_trace(jl.MARK_TRACE, metainterp_sd, None)
+        log.write_trace(self.trace)
+        if self.log_noopt:
+            metainterp_sd.logger_noopt.log_loop_from_trace(self.trace, memo=memo)
+
+        self.box_names_memo = memo
+        optimizations = build_opt_chain(self.enable_opts)
+        debug_start("jit-optimize")
+        try:
+            return self.optimize(metainterp_sd, jitdriver_sd, optimizations)
+        finally:
+            self.forget_optimization_info()
+            debug_stop("jit-optimize")
+
+
 class PreambleCompileData(CompileData):
     """
     This is the case of label() ops label()
@@ -54,10 +74,9 @@
     def optimize(self, metainterp_sd, jitdriver_sd, optimizations):
         from rpython.jit.metainterp.optimizeopt.unroll import UnrollOptimizer
         opt = UnrollOptimizer(metainterp_sd, jitdriver_sd, optimizations)
-        return opt.optimize_preamble(self.trace,
-                                        self.runtime_boxes,
-                                        self.call_pure_results,
-                                        self.box_names_memo)
+        return opt.optimize_preamble(
+            self.trace, self.runtime_boxes, self.call_pure_results,
+            self.box_names_memo)
 
 class SimpleCompileData(CompileData):
     """ This represents label() ops jump with no extra info associated with
@@ -72,7 +91,8 @@
 
     def optimize(self, metainterp_sd, jitdriver_sd, optimizations):
         from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer
-        from rpython.jit.metainterp.optimizeopt.bridgeopt import deserialize_optimizer_knowledge
+        from rpython.jit.metainterp.optimizeopt.bridgeopt \
+            import deserialize_optimizer_knowledge
         opt = Optimizer(metainterp_sd, jitdriver_sd, optimizations)
         traceiter = self.trace.get_iter()
         if self.resumestorage:
@@ -86,8 +106,9 @@
     """ This represents ops() with a jump at the end that goes to some
     loop, we need to deal with virtual state and inlining of short preamble
     """
-    def __init__(self, trace, runtime_boxes, resumestorage=None, call_pure_results=None,
-                 enable_opts=None, inline_short_preamble=False):
+    def __init__(self, trace, runtime_boxes, resumestorage=None,
+                 call_pure_results=None, enable_opts=None,
+                 inline_short_preamble=False):
         self.trace = trace
         self.runtime_boxes = runtime_boxes
         self.call_pure_results = call_pure_results
@@ -203,8 +224,6 @@
 
 def compile_simple_loop(metainterp, greenkey, trace, runtime_args, enable_opts,
                         cut_at):
-    from rpython.jit.metainterp.optimizeopt import optimize_trace
-
     jitdriver_sd = metainterp.jitdriver_sd
     metainterp_sd = metainterp.staticdata
     jitcell_token = make_jitcell_token(jitdriver_sd)
@@ -212,8 +231,8 @@
     data = SimpleCompileData(trace, call_pure_results=call_pure_results,
                              enable_opts=enable_opts)
     try:
-        loop_info, ops = optimize_trace(
-            metainterp_sd, jitdriver_sd, data, metainterp.box_names_memo)
+        loop_info, ops = data.optimize_trace(
+            metainterp_sd, jitdriver_sd, metainterp.box_names_memo)
     except InvalidLoop:
         metainterp_sd.jitlog.trace_aborted()
         trace.cut_at(cut_at)
@@ -242,8 +261,6 @@
     """Try to compile a new procedure by closing the current history back
     to the first operation.
     """
-    from rpython.jit.metainterp.optimizeopt import optimize_trace
-
     metainterp_sd = metainterp.staticdata
     jitdriver_sd = metainterp.jitdriver_sd
     history = metainterp.history
@@ -267,9 +284,8 @@
                                     call_pure_results=call_pure_results,
                                     enable_opts=enable_opts)
     try:
-        start_state, preamble_ops = optimize_trace(
-            metainterp_sd, jitdriver_sd, preamble_data,
-            metainterp.box_names_memo)
+        start_state, preamble_ops = preamble_data.optimize_trace(
+            metainterp_sd, jitdriver_sd, metainterp.box_names_memo)
     except InvalidLoop:
         metainterp_sd.jitlog.trace_aborted()
         history.cut(cut_at)
@@ -284,8 +300,8 @@
                                  call_pure_results=call_pure_results,
                                  enable_opts=enable_opts)
     try:
-        loop_info, loop_ops = optimize_trace(
-            metainterp_sd, jitdriver_sd, loop_data, metainterp.box_names_memo)
+        loop_info, loop_ops = loop_data.optimize_trace(
+            metainterp_sd, jitdriver_sd, metainterp.box_names_memo)
     except InvalidLoop:
         metainterp_sd.jitlog.trace_aborted()
         history.cut(cut_at)
@@ -332,8 +348,6 @@
     """Try to compile a new procedure by closing the current history back
     to the first operation.
     """
-    from rpython.jit.metainterp.optimizeopt import optimize_trace
-
     trace = metainterp.history.trace.cut_trace_from(start, inputargs)
     metainterp_sd = metainterp.staticdata
     jitdriver_sd = metainterp.jitdriver_sd
@@ -353,8 +367,8 @@
                                  call_pure_results=call_pure_results,
                                  enable_opts=enable_opts)
     try:
-        loop_info, loop_ops = optimize_trace(
-            metainterp_sd, jitdriver_sd, loop_data, metainterp.box_names_memo)
+        loop_info, loop_ops = loop_data.optimize_trace(
+            metainterp_sd, jitdriver_sd, metainterp.box_names_memo)
     except InvalidLoop:
         # Fall back on jumping directly to preamble
         history.cut(cut)
@@ -364,9 +378,8 @@
                                      enable_opts=enable_opts,
                                      inline_short_preamble=False)
         try:
-            loop_info, loop_ops = optimize_trace(
-                metainterp_sd, jitdriver_sd, loop_data,
-                metainterp.box_names_memo)
+            loop_info, loop_ops = loop_data.optimize_trace(
+                metainterp_sd, jitdriver_sd, metainterp.box_names_memo)
         except InvalidLoop:
             metainterp_sd.jitlog.trace_aborted()
             history.cut(cut)
@@ -1024,8 +1037,6 @@
     """Try to compile a new bridge leading from the beginning of the history
     to some existing place.
     """
-    from rpython.jit.metainterp.optimizeopt import optimize_trace
-
     # The history contains new operations to attach as the code for the
     # failure of 'resumekey.guard_op'.
     #
@@ -1060,8 +1071,8 @@
                                  call_pure_results=call_pure_results,
                                  enable_opts=enable_opts)
     try:
-        info, newops = optimize_trace(
-            metainterp_sd, jitdriver_sd, data, metainterp.box_names_memo)
+        info, newops = data.optimize_trace(
+            metainterp_sd, jitdriver_sd, metainterp.box_names_memo)
     except InvalidLoop:
         metainterp_sd.jitlog.trace_aborted()
         # XXX I am fairly convinced that optimize_bridge cannot actually raise
diff --git a/rpython/jit/metainterp/optimizeopt/__init__.py b/rpython/jit/metainterp/optimizeopt/__init__.py
--- a/rpython/jit/metainterp/optimizeopt/__init__.py
+++ b/rpython/jit/metainterp/optimizeopt/__init__.py
@@ -1,4 +1,3 @@
-from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer
 from rpython.jit.metainterp.optimizeopt.rewrite import OptRewrite
 from rpython.jit.metainterp.optimizeopt.intbounds import OptIntBounds
 from rpython.jit.metainterp.optimizeopt.virtualize import OptVirtualize
@@ -7,10 +6,8 @@
 from rpython.jit.metainterp.optimizeopt.simplify import OptSimplify
 from rpython.jit.metainterp.optimizeopt.pure import OptPure
 from rpython.jit.metainterp.optimizeopt.earlyforce import OptEarlyForce
-from rpython.rlib.rjitlog import rjitlog as jl
 from rpython.rlib.jit import PARAMETERS, ENABLE_ALL_OPTS
 from rpython.rlib.unroll import unrolling_iterable
-from rpython.rlib.debug import debug_start, debug_stop, debug_print
 
 
 ALL_OPTS = [('intbounds', OptIntBounds),
@@ -43,29 +40,5 @@
         optimizations.append(OptSimplify())
     return optimizations
 
-def _log_loop_from_trace(metainterp_sd, trace, memo=None, is_unrolled=False):
-    # mark that a new trace has been started
-    log = metainterp_sd.jitlog.log_trace(jl.MARK_TRACE, metainterp_sd, None)
-    log.write_trace(trace)
-    if not is_unrolled:
-        metainterp_sd.logger_noopt.log_loop_from_trace(trace, memo=memo)
-
-def optimize_trace(metainterp_sd, jitdriver_sd, compile_data, memo=None):
-    """Optimize loop.operations to remove internal overheadish operations.
-    """
-    if memo is None:
-        memo = {}
-    debug_start("jit-optimize")
-    try:
-        _log_loop_from_trace(metainterp_sd, compile_data.trace, memo,
-                             is_unrolled=not compile_data.log_noopt)
-        compile_data.box_names_memo = memo
-        optimizations = build_opt_chain(compile_data.enable_opts)
-        return compile_data.optimize(
-            metainterp_sd, jitdriver_sd, optimizations)
-    finally:
-        compile_data.forget_optimization_info()
-        debug_stop("jit-optimize")
-
 if __name__ == '__main__':
     print ALL_OPTS_NAMES
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -23,7 +23,6 @@
 from rpython.jit.metainterp.resoperation import (
     rop, ResOperation, InputArgRef, AbstractValue)
 from rpython.jit.metainterp.virtualref import VirtualRefInfo
-from rpython.jit.metainterp.optimizeopt import optimize_trace
 from rpython.jit.metainterp.optimizeopt.util import (
     sort_descrs, equaloplists, args_dict)
 
@@ -544,7 +543,7 @@
         compute_bitstrings(self.cpu.fetch_all_descrs())
         #
         compile_data.enable_opts = self.enable_opts
-        state = optimize_trace(metainterp_sd, None, compile_data)
+        state = compile_data.optimize_trace(metainterp_sd, None, {})
         return state
 
     def _convert_call_pure_results(self, d):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
@@ -853,7 +853,6 @@
     enable_opts = "intbounds:rewrite:virtualize:string:pure:earlyforce:heap:unroll"
 
     def _do_optimize_bridge(self, bridge, call_pure_results, values):
-        from rpython.jit.metainterp.optimizeopt import optimize_trace
         from rpython.jit.metainterp.optimizeopt.util import args_dict
 
         self.bridge = bridge
@@ -871,7 +870,7 @@
         data = compile.BridgeCompileData(trace, runtime_boxes,
             enable_opts=self.enable_opts, inline_short_preamble=True)
 
-        info, newops = optimize_trace(metainterp_sd, None, data)
+        info, newops = data.optimize_trace(metainterp_sd, None, {})
         if info.final():
             bridge.operations = newops
             bridge.inputargs = info.inputargs

From pypy.commits at gmail.com  Thu May  9 12:35:59 2019
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 09 May 2019 09:35:59 -0700 (PDT)
Subject: [pypy-commit] pypy optimizeopt-cleanup: Create
 Optimizer.optimize_loop() for consistency
Message-ID: <5cd456ef.1c69fb81.6cd94.1e84@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: optimizeopt-cleanup
Changeset: r96586:612d770461f7
Date: 2019-05-09 17:35 +0100
http://bitbucket.org/pypy/pypy/changeset/612d770461f7/

Log:	Create Optimizer.optimize_loop() for consistency

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -91,16 +91,9 @@
 
     def optimize(self, metainterp_sd, jitdriver_sd, optimizations):
         from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer
-        from rpython.jit.metainterp.optimizeopt.bridgeopt \
-            import deserialize_optimizer_knowledge
         opt = Optimizer(metainterp_sd, jitdriver_sd, optimizations)
-        traceiter = self.trace.get_iter()
-        if self.resumestorage:
-            frontend_inputargs = self.trace.inputargs
-            deserialize_optimizer_knowledge(opt, self.resumestorage,
-                                            frontend_inputargs,
-                                            traceiter.inputargs)
-        return opt.propagate_all_forward(traceiter, self.call_pure_results)
+        return opt.optimize_loop(
+            self.trace, self.resumestorage, self.call_pure_results)
 
 class BridgeCompileData(CompileData):
     """ This represents ops() with a jump at the end that goes to some
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -6,6 +6,8 @@
     IntBound, ConstIntBound, MININT, MAXINT, IntUnbounded)
 from rpython.jit.metainterp.optimizeopt.util import (
     make_dispatcher_method, get_box_replacement)
+from rpython.jit.metainterp.optimizeopt.bridgeopt import (
+    deserialize_optimizer_knowledge)
 from rpython.jit.metainterp.resoperation import (
     rop, AbstractResOp, GuardResOp, OpHelpers)
 from rpython.jit.metainterp.optimizeopt import info
@@ -296,6 +298,14 @@
 
         self.optimizations = optimizations
 
+    def optimize_loop(self, trace, resumestorage, call_pure_results):
+        traceiter = trace.get_iter()
+        if resumestorage:
+            frontend_inputargs = trace.inputargs
+            deserialize_optimizer_knowledge(
+                self, resumestorage, frontend_inputargs, traceiter.inputargs)
+        return self.propagate_all_forward(traceiter, call_pure_results)
+
     def force_op_from_preamble(self, op):
         return op
 

From pypy.commits at gmail.com  Thu May  9 15:10:54 2019
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 09 May 2019 12:10:54 -0700 (PDT)
Subject: [pypy-commit] pypy optimizeopt-cleanup: Match class names to their
 base classes
Message-ID: <5cd47b3e.1c69fb81.228a0.5cbc@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: optimizeopt-cleanup
Changeset: r96587:55c98086c509
Date: 2019-05-09 17:41 +0100
http://bitbucket.org/pypy/pypy/changeset/55c98086c509/

Log:	Match class names to their base classes

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -72,8 +72,8 @@
         self.runtime_boxes = runtime_boxes
 
     def optimize(self, metainterp_sd, jitdriver_sd, optimizations):
-        from rpython.jit.metainterp.optimizeopt.unroll import UnrollOptimizer
-        opt = UnrollOptimizer(metainterp_sd, jitdriver_sd, optimizations)
+        from rpython.jit.metainterp.optimizeopt.unroll import OptUnroll
+        opt = OptUnroll(metainterp_sd, jitdriver_sd, optimizations)
         return opt.optimize_preamble(
             self.trace, self.runtime_boxes, self.call_pure_results,
             self.box_names_memo)
@@ -110,9 +110,9 @@
         self.resumestorage = resumestorage
 
     def optimize(self, metainterp_sd, jitdriver_sd, optimizations):
-        from rpython.jit.metainterp.optimizeopt.unroll import UnrollOptimizer
+        from rpython.jit.metainterp.optimizeopt.unroll import OptUnroll
 
-        opt = UnrollOptimizer(metainterp_sd, jitdriver_sd, optimizations)
+        opt = OptUnroll(metainterp_sd, jitdriver_sd, optimizations)
         return opt.optimize_bridge(self.trace, self.runtime_boxes,
                                    self.call_pure_results,
                                    self.inline_short_preamble,
@@ -136,8 +136,8 @@
         self.inline_short_preamble = inline_short_preamble
 
     def optimize(self, metainterp_sd, jitdriver_sd, optimizations):
-        from rpython.jit.metainterp.optimizeopt.unroll import UnrollOptimizer
-        opt = UnrollOptimizer(metainterp_sd, jitdriver_sd, optimizations)
+        from rpython.jit.metainterp.optimizeopt.unroll import OptUnroll
+        opt = OptUnroll(metainterp_sd, jitdriver_sd, optimizations)
         return opt.optimize_peeled_loop(self.trace, self.celltoken, self.state,
             self.call_pure_results, self.inline_short_preamble)
 
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -16,7 +16,7 @@
 from rpython.rlib.debug import debug_print, debug_start, debug_stop,\
      have_debug_prints
 
-class UnrollableOptimizer(Optimizer):
+class UnrollOptimizer(Optimizer):
     def force_op_from_preamble(self, preamble_op):
         if isinstance(preamble_op, PreambleOp):
             if self.optunroll.short_preamble_producer is None:
@@ -99,7 +99,7 @@
             op.set_forwarded(preamble_info._const)
 
 
-class UnrollOptimizer(Optimization):
+class OptUnroll(Optimization):
     """Unroll the loop into two iterations. The first one will
     become the preamble or entry bridge (don't think there is a
     distinction anymore)"""
@@ -107,7 +107,7 @@
     short_preamble_producer = None
 
     def __init__(self, metainterp_sd, jitdriver_sd, optimizations):
-        self.optimizer = UnrollableOptimizer(metainterp_sd, jitdriver_sd,
+        self.optimizer = UnrollOptimizer(metainterp_sd, jitdriver_sd,
                                              optimizations)
         self.optimizer.optunroll = self
 

From pypy.commits at gmail.com  Thu May  9 15:10:56 2019
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 09 May 2019 12:10:56 -0700 (PDT)
Subject: [pypy-commit] pypy optimizeopt-cleanup: Move some methods from
 OptUnroll to UnrollOptimizer
Message-ID: <5cd47b40.1c69fb81.92ab3.7aa0@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: optimizeopt-cleanup
Changeset: r96588:cfa068539b62
Date: 2019-05-09 19:46 +0100
http://bitbucket.org/pypy/pypy/changeset/cfa068539b62/

Log:	Move some methods from OptUnroll to UnrollOptimizer

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -72,8 +72,8 @@
         self.runtime_boxes = runtime_boxes
 
     def optimize(self, metainterp_sd, jitdriver_sd, optimizations):
-        from rpython.jit.metainterp.optimizeopt.unroll import OptUnroll
-        opt = OptUnroll(metainterp_sd, jitdriver_sd, optimizations)
+        from rpython.jit.metainterp.optimizeopt.unroll import UnrollOptimizer
+        opt = UnrollOptimizer(metainterp_sd, jitdriver_sd, optimizations)
         return opt.optimize_preamble(
             self.trace, self.runtime_boxes, self.call_pure_results,
             self.box_names_memo)
@@ -110,9 +110,9 @@
         self.resumestorage = resumestorage
 
     def optimize(self, metainterp_sd, jitdriver_sd, optimizations):
-        from rpython.jit.metainterp.optimizeopt.unroll import OptUnroll
+        from rpython.jit.metainterp.optimizeopt.unroll import UnrollOptimizer
 
-        opt = OptUnroll(metainterp_sd, jitdriver_sd, optimizations)
+        opt = UnrollOptimizer(metainterp_sd, jitdriver_sd, optimizations)
         return opt.optimize_bridge(self.trace, self.runtime_boxes,
                                    self.call_pure_results,
                                    self.inline_short_preamble,
@@ -136,8 +136,8 @@
         self.inline_short_preamble = inline_short_preamble
 
     def optimize(self, metainterp_sd, jitdriver_sd, optimizations):
-        from rpython.jit.metainterp.optimizeopt.unroll import OptUnroll
-        opt = OptUnroll(metainterp_sd, jitdriver_sd, optimizations)
+        from rpython.jit.metainterp.optimizeopt.unroll import UnrollOptimizer
+        opt = UnrollOptimizer(metainterp_sd, jitdriver_sd, optimizations)
         return opt.optimize_peeled_loop(self.trace, self.celltoken, self.state,
             self.call_pure_results, self.inline_short_preamble)
 
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -17,6 +17,11 @@
      have_debug_prints
 
 class UnrollOptimizer(Optimizer):
+    def __init__(self, metainterp_sd, jitdriver_sd, optimizations):
+        Optimizer.__init__(self, metainterp_sd, jitdriver_sd, optimizations)
+        self.optunroll = OptUnroll()
+        self.optunroll.optimizer = self
+
     def force_op_from_preamble(self, preamble_op):
         if isinstance(preamble_op, PreambleOp):
             if self.optunroll.short_preamble_producer is None:
@@ -98,6 +103,148 @@
         elif isinstance(preamble_info, info.FloatConstInfo):
             op.set_forwarded(preamble_info._const)
 
+    def optimize_preamble(self, trace, runtime_boxes, call_pure_results, memo):
+        info, newops = self.propagate_all_forward(
+            trace.get_iter(), call_pure_results, flush=False)
+        exported_state = self.optunroll.export_state(info.jump_op.getarglist(),
+                                           info.inputargs,
+                                           runtime_boxes, memo)
+        exported_state.quasi_immutable_deps = info.quasi_immutable_deps
+        # we need to absolutely make sure that we've cleaned up all
+        # the optimization info
+        self._clean_optimization_info(self._newoperations)
+        return exported_state, self._newoperations
+
+    def optimize_peeled_loop(self, trace, celltoken, state,
+                             call_pure_results, inline_short_preamble=True):
+        trace = trace.get_iter()
+        try:
+            label_args = self.optunroll.import_state(trace.inputargs, state)
+        except VirtualStatesCantMatch:
+            raise InvalidLoop("Cannot import state, virtual states don't match")
+        self.optunroll.potential_extra_ops = {}
+        try:
+            info, _ = self.propagate_all_forward(
+                trace, call_pure_results, flush=False)
+        except SpeculativeError:
+            raise InvalidLoop("Speculative heap access would be ill-typed")
+        end_jump = info.jump_op
+        label_op = ResOperation(rop.LABEL, label_args, descr=celltoken)
+        for a in end_jump.getarglist():
+            self.force_box_for_end_of_preamble(get_box_replacement(a))
+        current_vs = self.optunroll.get_virtual_state(end_jump.getarglist())
+        # pick the vs we want to jump to
+        assert isinstance(celltoken, JitCellToken)
+
+        target_virtual_state = self.optunroll.pick_virtual_state(
+            current_vs, state.virtual_state, celltoken.target_tokens)
+        # force the boxes for virtual state to match
+        try:
+            args = target_virtual_state.make_inputargs(
+               [get_box_replacement(x) for x in end_jump.getarglist()],
+               self, force_boxes=True)
+            for arg in args:
+                if arg is not None:
+                    self.force_box(arg)
+        except VirtualStatesCantMatch:
+            raise InvalidLoop("Virtual states did not match "
+                              "after picking the virtual state, when forcing"
+                              " boxes")
+        extra_same_as = self.optunroll.short_preamble_producer.extra_same_as[:]
+        target_token = self.optunroll.finalize_short_preamble(label_op,
+                                                    state.virtual_state)
+        label_op.setdescr(target_token)
+
+        if not inline_short_preamble:
+            self.optunroll.jump_to_preamble(celltoken, end_jump, info)
+            return (UnrollInfo(target_token, label_op, extra_same_as,
+                               self.quasi_immutable_deps),
+                    self._newoperations)
+
+        try:
+            new_virtual_state = self.optunroll.jump_to_existing_trace(
+                    end_jump, label_op, state.runtime_boxes, force_boxes=False)
+        except InvalidLoop:
+            # inlining short preamble failed, jump to preamble
+            self.optunroll.jump_to_preamble(celltoken, end_jump, info)
+            return (UnrollInfo(target_token, label_op, extra_same_as,
+                               self.quasi_immutable_deps),
+                    self._newoperations)
+
+        if new_virtual_state is not None:
+            # Attempt to force virtual boxes in order to avoid jumping
+            # to the preamble.
+            try:
+                new_virtual_state = self.optunroll.jump_to_existing_trace(
+                        end_jump, label_op, state.runtime_boxes, force_boxes=True)
+            except InvalidLoop:
+                pass
+
+        if new_virtual_state is not None:
+            self.optunroll.jump_to_preamble(celltoken, end_jump, info)
+            return (UnrollInfo(target_token, label_op, extra_same_as,
+                               self.quasi_immutable_deps),
+                    self._newoperations)
+
+        self.optunroll.disable_retracing_if_max_retrace_guards(
+            self._newoperations, target_token)
+
+        return (UnrollInfo(target_token, label_op, extra_same_as,
+                           self.quasi_immutable_deps),
+                self._newoperations)
+
+    def optimize_bridge(self, trace, runtime_boxes, call_pure_results,
+                        inline_short_preamble, box_names_memo, resumestorage):
+        from rpython.jit.metainterp.optimizeopt.bridgeopt import deserialize_optimizer_knowledge
+        frontend_inputargs = trace.inputargs
+        trace = trace.get_iter()
+        self.optunroll._check_no_forwarding([trace.inputargs])
+        if resumestorage:
+            deserialize_optimizer_knowledge(self,
+                                            resumestorage, frontend_inputargs,
+                                            trace.inputargs)
+        info, ops = self.propagate_all_forward(trace,
+            call_pure_results, False)
+        jump_op = info.jump_op
+        cell_token = jump_op.getdescr()
+        assert isinstance(cell_token, JitCellToken)
+        if not inline_short_preamble or len(cell_token.target_tokens) == 1:
+            return self.optunroll.jump_to_preamble(cell_token, jump_op, info)
+        # force all the information that does not go to the short
+        # preamble at all
+        self.flush()
+        for a in jump_op.getarglist():
+            self.force_box_for_end_of_preamble(a)
+        try:
+            vs = self.optunroll.jump_to_existing_trace(jump_op, None, runtime_boxes,
+                                             force_boxes=False)
+        except InvalidLoop:
+            return self.optunroll.jump_to_preamble(cell_token, jump_op, info)
+        if vs is None:
+            return info, self._newoperations[:]
+        warmrunnerdescr = self.metainterp_sd.warmrunnerdesc
+        limit = warmrunnerdescr.memory_manager.retrace_limit
+        if cell_token.retraced_count < limit:
+            cell_token.retraced_count += 1
+            debug_print('Retracing (%d/%d)' % (cell_token.retraced_count, limit))
+        else:
+            # Try forcing boxes to avoid jumping to the preamble
+            try:
+                vs = self.optunroll.jump_to_existing_trace(jump_op, None, runtime_boxes,
+                                                 force_boxes=True)
+            except InvalidLoop:
+                pass
+            if vs is None:
+                return info, self._newoperations[:]
+            debug_print("Retrace count reached, jumping to preamble")
+            return self.optunroll.jump_to_preamble(cell_token, jump_op, info)
+        exported_state = self.optunroll.export_state(info.jump_op.getarglist(),
+                                           info.inputargs, runtime_boxes,
+                                           box_names_memo)
+        exported_state.quasi_immutable_deps = self.quasi_immutable_deps
+        self._clean_optimization_info(self._newoperations)
+        return exported_state, self._newoperations
+
 
 class OptUnroll(Optimization):
     """Unroll the loop into two iterations. The first one will
@@ -106,11 +253,6 @@
 
     short_preamble_producer = None
 
-    def __init__(self, metainterp_sd, jitdriver_sd, optimizations):
-        self.optimizer = UnrollOptimizer(metainterp_sd, jitdriver_sd,
-                                             optimizations)
-        self.optimizer.optunroll = self
-
     def get_virtual_state(self, args):
         modifier = VirtualStateConstructor(self.optimizer)
         return modifier.get_virtual_state(args)
@@ -123,99 +265,6 @@
             assert not self.optimizer._newoperations
 
 
-    def optimize_preamble(self, trace, runtime_boxes, call_pure_results, memo):
-        info, newops = self.optimizer.propagate_all_forward(
-            trace.get_iter(), call_pure_results, flush=False)
-        exported_state = self.export_state(info.jump_op.getarglist(),
-                                           info.inputargs,
-                                           runtime_boxes, memo)
-        exported_state.quasi_immutable_deps = info.quasi_immutable_deps
-        # we need to absolutely make sure that we've cleaned up all
-        # the optimization info
-        self.optimizer._clean_optimization_info(self.optimizer._newoperations)
-        return exported_state, self.optimizer._newoperations
-
-    def optimize_peeled_loop(self, trace, celltoken, state,
-                             call_pure_results, inline_short_preamble=True):
-        trace = trace.get_iter()
-        try:
-            label_args = self.import_state(trace.inputargs, state)
-        except VirtualStatesCantMatch:
-            raise InvalidLoop("Cannot import state, virtual states don't match")
-        self.potential_extra_ops = {}
-        try:
-            info, _ = self.optimizer.propagate_all_forward(
-                trace, call_pure_results, flush=False)
-        except SpeculativeError:
-            raise InvalidLoop("Speculative heap access would be ill-typed")
-        end_jump = info.jump_op
-        label_op = ResOperation(rop.LABEL, label_args,
-                                descr=celltoken)
-        for a in end_jump.getarglist():
-            self.optimizer.force_box_for_end_of_preamble(
-                get_box_replacement(a))
-        current_vs = self.get_virtual_state(end_jump.getarglist())
-        # pick the vs we want to jump to
-        assert isinstance(celltoken, JitCellToken)
-
-        target_virtual_state = self.pick_virtual_state(current_vs,
-                                                       state.virtual_state,
-                                                celltoken.target_tokens)
-        # force the boxes for virtual state to match
-        try:
-            args = target_virtual_state.make_inputargs(
-               [get_box_replacement(x) for x in end_jump.getarglist()],
-               self.optimizer, force_boxes=True)
-            for arg in args:
-                if arg is not None:
-                    self.optimizer.force_box(arg)
-        except VirtualStatesCantMatch:
-            raise InvalidLoop("Virtual states did not match "
-                              "after picking the virtual state, when forcing"
-                              " boxes")
-        extra_same_as = self.short_preamble_producer.extra_same_as[:]
-        target_token = self.finalize_short_preamble(label_op,
-                                                    state.virtual_state)
-        label_op.setdescr(target_token)
-
-        if not inline_short_preamble:
-            self.jump_to_preamble(celltoken, end_jump, info)
-            return (UnrollInfo(target_token, label_op, extra_same_as,
-                               self.optimizer.quasi_immutable_deps),
-                    self.optimizer._newoperations)
-
-        try:
-            new_virtual_state = self.jump_to_existing_trace(
-                    end_jump, label_op, state.runtime_boxes, force_boxes=False)
-        except InvalidLoop:
-            # inlining short preamble failed, jump to preamble
-            self.jump_to_preamble(celltoken, end_jump, info)
-            return (UnrollInfo(target_token, label_op, extra_same_as,
-                               self.optimizer.quasi_immutable_deps),
-                    self.optimizer._newoperations)
-
-        if new_virtual_state is not None:
-            # Attempt to force virtual boxes in order to avoid jumping
-            # to the preamble.
-            try:
-                new_virtual_state = self.jump_to_existing_trace(
-                        end_jump, label_op, state.runtime_boxes, force_boxes=True)
-            except InvalidLoop:
-                pass
-
-        if new_virtual_state is not None:
-            self.jump_to_preamble(celltoken, end_jump, info)
-            return (UnrollInfo(target_token, label_op, extra_same_as,
-                               self.optimizer.quasi_immutable_deps),
-                    self.optimizer._newoperations)
-
-        self.disable_retracing_if_max_retrace_guards(
-            self.optimizer._newoperations, target_token)
-
-        return (UnrollInfo(target_token, label_op, extra_same_as,
-                           self.optimizer.quasi_immutable_deps),
-                self.optimizer._newoperations)
-
     def disable_retracing_if_max_retrace_guards(self, ops, target_token):
         maxguards = self.optimizer.metainterp_sd.warmrunnerdesc.memory_manager.max_retrace_guards
         count = 0
@@ -236,58 +285,6 @@
                 return token.virtual_state
         return label_vs
 
-    def optimize_bridge(self, trace, runtime_boxes, call_pure_results,
-                        inline_short_preamble, box_names_memo, resumestorage):
-        from rpython.jit.metainterp.optimizeopt.bridgeopt import deserialize_optimizer_knowledge
-        frontend_inputargs = trace.inputargs
-        trace = trace.get_iter()
-        self._check_no_forwarding([trace.inputargs])
-        if resumestorage:
-            deserialize_optimizer_knowledge(self.optimizer,
-                                            resumestorage, frontend_inputargs,
-                                            trace.inputargs)
-        info, ops = self.optimizer.propagate_all_forward(trace,
-            call_pure_results, False)
-        jump_op = info.jump_op
-        cell_token = jump_op.getdescr()
-        assert isinstance(cell_token, JitCellToken)
-        if not inline_short_preamble or len(cell_token.target_tokens) == 1:
-            return self.jump_to_preamble(cell_token, jump_op, info)
-        # force all the information that does not go to the short
-        # preamble at all
-        self.optimizer.flush()
-        for a in jump_op.getarglist():
-            self.optimizer.force_box_for_end_of_preamble(a)
-        try:
-            vs = self.jump_to_existing_trace(jump_op, None, runtime_boxes,
-                                             force_boxes=False)
-        except InvalidLoop:
-            return self.jump_to_preamble(cell_token, jump_op, info)
-        if vs is None:
-            return info, self.optimizer._newoperations[:]
-        warmrunnerdescr = self.optimizer.metainterp_sd.warmrunnerdesc
-        limit = warmrunnerdescr.memory_manager.retrace_limit
-        if cell_token.retraced_count < limit:
-            cell_token.retraced_count += 1
-            debug_print('Retracing (%d/%d)' % (cell_token.retraced_count, limit))
-        else:
-            # Try forcing boxes to avoid jumping to the preamble
-            try:
-                vs = self.jump_to_existing_trace(jump_op, None, runtime_boxes,
-                                                 force_boxes=True)
-            except InvalidLoop:
-                pass
-            if vs is None:
-                return info, self.optimizer._newoperations[:]
-            debug_print("Retrace count reached, jumping to preamble")
-            return self.jump_to_preamble(cell_token, jump_op, info)
-        exported_state = self.export_state(info.jump_op.getarglist(),
-                                           info.inputargs, runtime_boxes,
-                                           box_names_memo)
-        exported_state.quasi_immutable_deps = self.optimizer.quasi_immutable_deps
-        self.optimizer._clean_optimization_info(self.optimizer._newoperations)
-        return exported_state, self.optimizer._newoperations
-
     def finalize_short_preamble(self, label_op, virtual_state):
         sb = self.short_preamble_producer
         self.optimizer._clean_optimization_info(sb.short_inputargs)

From pypy.commits at gmail.com  Thu May  9 15:10:58 2019
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 09 May 2019 12:10:58 -0700 (PDT)
Subject: [pypy-commit] pypy optimizeopt-cleanup: kill unused parameters
Message-ID: <5cd47b42.1c69fb81.8b2c2.6e57@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: optimizeopt-cleanup
Changeset: r96589:b4e53e717a3b
Date: 2019-05-09 19:47 +0100
http://bitbucket.org/pypy/pypy/changeset/b4e53e717a3b/

Log:	kill unused parameters

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -135,7 +135,7 @@
             opinfo.mark_last_guard(self.optimizer)
         return opinfo
 
-    def getptrinfo(self, op, is_object=False):
+    def getptrinfo(self, op):
         if op.type == 'i':
             return self.getrawptrinfo(op)
         elif op.type == 'f':
@@ -157,7 +157,7 @@
             return True
         return False
 
-    def getrawptrinfo(self, op, create=False, is_object=False):
+    def getrawptrinfo(self, op):
         assert op.type == 'i'
         op = get_box_replacement(op)
         assert op.type == 'i'
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -252,7 +252,7 @@
         return self.emit(op)
 
     def optimize_INT_ADD(self, op):
-        opinfo = self.getrawptrinfo(op.getarg(0), create=False)
+        opinfo = self.getrawptrinfo(op.getarg(0))
         offsetbox = self.get_constant_box(op.getarg(1))
         if opinfo and opinfo.is_virtual() and offsetbox is not None:
             offset = offsetbox.getint()
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -241,7 +241,7 @@
                     info = optimizer.getptrinfo(box)
                     is_virtual = (info is not None and info.is_virtual())
                 if box.type == 'i':
-                    info = optimizer.getrawptrinfo(box, create=False)
+                    info = optimizer.getrawptrinfo(box)
                     is_virtual = (info is not None and info.is_virtual())
                 if is_virtual:
                     tagged = tag(num_virtuals, TAGVIRTUAL)
diff --git a/rpython/jit/metainterp/test/test_resume.py b/rpython/jit/metainterp/test/test_resume.py
--- a/rpython/jit/metainterp/test/test_resume.py
+++ b/rpython/jit/metainterp/test/test_resume.py
@@ -51,11 +51,11 @@
             op = op.get_forwarded()
         return op
 
-    def getrawptrinfo(self, op, create=True):
+    def getrawptrinfo(self, op):
         op = self.get_box_replacement(op)
         return op.get_forwarded()
 
-    def getptrinfo(self, op, create=True):
+    def getptrinfo(self, op):
         op = self.get_box_replacement(op)
         return op.get_forwarded()
 

From pypy.commits at gmail.com  Thu May  9 15:16:26 2019
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 09 May 2019 12:16:26 -0700 (PDT)
Subject: [pypy-commit] pypy optimizeopt-cleanup: fix
Message-ID: <5cd47c8a.1c69fb81.6612b.7574@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: optimizeopt-cleanup
Changeset: r96590:ed3e463f7b3f
Date: 2019-05-09 20:15 +0100
http://bitbucket.org/pypy/pypy/changeset/ed3e463f7b3f/

Log:	fix

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -164,7 +164,7 @@
         if isinstance(op, ConstInt):
             return info.ConstPtrInfo(op)
         fw = op.get_forwarded()
-        if isinstance(fw, IntBound) and not create:
+        if isinstance(fw, IntBound):
             return None
         if fw is not None:
             if isinstance(fw, info.AbstractRawPtrInfo):

From pypy.commits at gmail.com  Fri May 10 02:13:31 2019
From: pypy.commits at gmail.com (stevie_92)
Date: Thu, 09 May 2019 23:13:31 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-gc-cycle: Fixed some (de)allocation bugs
 in existing and derived gc cpyext-classes
Message-ID: <5cd5168b.1c69fb81.b8e89.ae1a@mx.google.com>

Author: Stefan Beyer <home at sbeyer.at>
Branch: cpyext-gc-cycle
Changeset: r96591:0dff3316692e
Date: 2019-05-09 23:32 +0200
http://bitbucket.org/pypy/pypy/changeset/0dff3316692e/

Log:	Fixed some (de)allocation bugs in existing and derived gc cpyext-
	classes

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -129,7 +129,7 @@
 Py_TPFLAGS_READY Py_TPFLAGS_READYING
 METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE Py_MAX_FMT
 METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O
-Py_TPFLAGS_HEAPTYPE
+Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_GC
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_MAX_NDIMS
 Py_CLEANUP_SUPPORTED
 PyBUF_FORMAT PyBUF_ND PyBUF_STRIDES PyBUF_WRITABLE PyBUF_SIMPLE PyBUF_WRITE
@@ -1182,6 +1182,9 @@
         [Py_ssize_t], PyObject,
         compilation_info=eci,
         _nowrapper=True)
+    state.C._PyPy_tuple_free = rffi.llexternal(
+        '_PyPy_tuple_free', [rffi.VOIDP], lltype.Void,
+        compilation_info=eci, _nowrapper=True)
     state.C._PyPy_tuple_dealloc = rffi.llexternal(
         '_PyPy_tuple_dealloc', [PyObject], lltype.Void,
         compilation_info=eci, _nowrapper=True)
diff --git a/pypy/module/cpyext/include/tupleobject.h b/pypy/module/cpyext/include/tupleobject.h
--- a/pypy/module/cpyext/include/tupleobject.h
+++ b/pypy/module/cpyext/include/tupleobject.h
@@ -18,6 +18,7 @@
 
 PyAPI_FUNC(PyObject *) PyTuple_New(Py_ssize_t size);
 PyAPI_FUNC(void) _PyPy_tuple_dealloc(PyObject *);
+PyAPI_FUNC(void) _PyPy_tuple_free(void *);
 
 /* defined in varargswrapper.c */
 PyAPI_FUNC(PyObject *) PyTuple_Pack(Py_ssize_t, ...);
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -7,7 +7,7 @@
 from pypy.module.cpyext.api import (
     cpython_api, bootstrap_function, PyObject, PyObjectP, ADDR,
     CANNOT_FAIL, Py_TPFLAGS_HEAPTYPE, PyTypeObjectPtr, is_PyObject,
-    PyVarObject, Py_ssize_t, init_function, cts)
+    PyVarObject, Py_ssize_t, init_function, cts, Py_TPFLAGS_HAVE_GC)
 from pypy.module.cpyext.state import State
 from pypy.objspace.std.typeobject import W_TypeObject
 from pypy.objspace.std.noneobject import W_NoneObject
@@ -77,37 +77,17 @@
         state = space.fromcache(State)
         return state.C._PyPy_subtype_dealloc
 
-    # CCC port to C
+    def get_free(self, space):
+        state = space.fromcache(State)
+        return state.C.PyObject_Free
+
     def allocate(self, space, w_type, itemcount=0, immortal=False):
-        # typically called from PyType_GenericAlloc via typedescr.allocate
-        # this returns a PyObject with ob_refcnt == 1.
-
-        pytype = as_pyobj(space, w_type)
-        pytype = rffi.cast(PyTypeObjectPtr, pytype)
-        assert pytype
-        # Don't increase refcount for non-heaptypes
-        flags = rffi.cast(lltype.Signed, pytype.c_tp_flags)
-        if flags & Py_TPFLAGS_HEAPTYPE:
-            incref(space, pytype)
-
-        if pytype:
-            size = pytype.c_tp_basicsize
-        else:
-            size = rffi.sizeof(self.basestruct)
-        if pytype.c_tp_itemsize:
-            size += itemcount * pytype.c_tp_itemsize
-        assert size >= rffi.sizeof(PyObject.TO)
-        buf = lltype.malloc(rffi.VOIDP.TO, size,
-                            flavor='raw', zero=True,
-                            add_memory_pressure=True, immortal=immortal)
-        pyobj = rffi.cast(PyObject, buf)
-        if pytype.c_tp_itemsize:
-            pyvarobj = rffi.cast(PyVarObject, pyobj)
-            pyvarobj.c_ob_size = itemcount
-        pyobj.c_ob_refcnt = 1
-        #pyobj.c_ob_pypy_link should get assigned very quickly
-        pyobj.c_ob_type = pytype
-        return pyobj
+        state = space.fromcache(State)
+        ob_type = rffi.cast(PyTypeObjectPtr, as_pyobj(space, w_type))
+        ptup = state.ccall("PyType_GenericAlloc", ob_type, itemcount)
+        if not ptup:
+            state.check_and_raise_exception(always=True)
+        return ptup
 
     def attach(self, space, pyobj, w_obj, w_userdata=None):
         pass
@@ -140,6 +120,7 @@
     attach    : Function called to tie a raw structure to a pypy object
     realize   : Function called to create a pypy object from a raw struct
     dealloc   : a @slot_function(), similar to PyObject_dealloc
+    free      : a @slot_function(), similar to PyObject_free
     """
 
     tp_basestruct = kw.pop('basestruct', PyObject.TO)
@@ -147,6 +128,7 @@
     tp_attach     = kw.pop('attach', None)
     tp_realize    = kw.pop('realize', None)
     tp_dealloc    = kw.pop('dealloc', None)
+    tp_free       = kw.pop('free', None)
     assert not kw, "Extra arguments to make_typedescr"
 
     null_dealloc = lltype.nullptr(lltype.FuncType([PyObject], lltype.Void))
@@ -166,6 +148,10 @@
             def get_dealloc(self, space):
                 return tp_dealloc
 
+        if tp_free:
+            def get_free(self, space):
+                return tp_free
+
         if tp_attach:
             def attach(self, space, pyobj, w_obj, w_userdata=None):
                 tp_attach(space, pyobj, w_obj, w_userdata)
diff --git a/pypy/module/cpyext/src/object.c b/pypy/module/cpyext/src/object.c
--- a/pypy/module/cpyext/src/object.c
+++ b/pypy/module/cpyext/src/object.c
@@ -132,7 +132,10 @@
 PyObject *
 PyType_GenericAlloc(PyTypeObject *type, Py_ssize_t nitems)
 {
-    return (PyObject*)_PyObject_NewVar(type, nitems);
+    if (PyType_IS_GC(type))
+        return (PyObject*)_PyObject_GC_NewVar(type, nitems);
+    else
+        return (PyObject*)_PyObject_NewVar(type, nitems);
 }
 
 PyObject *
diff --git a/pypy/module/cpyext/src/tupleobject.c b/pypy/module/cpyext/src/tupleobject.c
--- a/pypy/module/cpyext/src/tupleobject.c
+++ b/pypy/module/cpyext/src/tupleobject.c
@@ -89,3 +89,9 @@
 done:
     Py_TRASHCAN_SAFE_END(op)
 }
+
+void
+_PyPy_tuple_free(void *obj)
+{
+    PyObject_GC_Del(obj);
+}
\ No newline at end of file
diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c
--- a/pypy/module/cpyext/test/foo.c
+++ b/pypy/module/cpyext/test/foo.c
@@ -822,7 +822,7 @@
     if (PyType_Ready(&UnicodeSubtype3) < 0)
         INITERROR;
 
-    TupleLike.tp_flags = Py_TPFLAGS_DEFAULT;
+    TupleLike.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC;
     TupleLike.tp_base = &PyTuple_Type;
     if (PyType_Ready(&TupleLike) < 0)
         INITERROR;
diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py
--- a/pypy/module/cpyext/tupleobject.py
+++ b/pypy/module/cpyext/tupleobject.py
@@ -4,10 +4,11 @@
 from rpython.rlib.debug import fatalerror_notb
 from pypy.module.cpyext.api import (
     cpython_api, Py_ssize_t, build_type_checkers_flags,
-    PyVarObjectFields, cpython_struct, bootstrap_function, slot_function)
+    PyVarObjectFields, cpython_struct, bootstrap_function, slot_function,
+    PyTypeObjectPtr)
 from pypy.module.cpyext.pyobject import (
     PyObject, PyObjectP, make_ref, from_ref, decref, incref, BaseCpyTypedescr,
-    track_reference, make_typedescr, get_typedescr, pyobj_has_w_obj)
+    track_reference, make_typedescr, get_typedescr, pyobj_has_w_obj, as_pyobj)
 from pypy.module.cpyext.state import State
 from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall
 from pypy.objspace.std.tupleobject import W_TupleObject
@@ -44,7 +45,8 @@
                    attach=tuple_attach,
                    alloc=tuple_alloc,
                    dealloc=state.C._PyPy_tuple_dealloc,
-                   realize=tuple_realize)
+                   realize=tuple_realize,
+                   free=state.C._PyPy_tuple_free)
 
 PyTuple_Check, PyTuple_CheckExact = build_type_checkers_flags("Tuple")
 
@@ -65,7 +67,13 @@
             state.check_and_raise_exception(always=True)
         return ptup
     else:
-        return BaseCpyTypedescr.allocate(typedescr, space, w_type, itemcount)
+        if not we_are_translated() and itemcount == _BAD_ITEMCOUNT:
+            itemcount = -42
+        ob_type = rffi.cast(PyTypeObjectPtr, as_pyobj(space, w_type))
+        ptup = state.ccall("PyType_GenericAlloc", ob_type, itemcount)
+        if not ptup:
+            state.check_and_raise_exception(always=True)
+        return ptup
 
 def tuple_attach(space, py_obj, w_obj, w_userdata=None):
     """
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -20,7 +20,7 @@
     Py_TPFLAGS_DICT_SUBCLASS, Py_TPFLAGS_BASE_EXC_SUBCLASS,
     Py_TPFLAGS_TYPE_SUBCLASS,
     Py_TPFLAGS_BYTES_SUBCLASS,
-    Py_TPPYPYFLAGS_FLOAT_SUBCLASS,
+    Py_TPPYPYFLAGS_FLOAT_SUBCLASS, Py_TPFLAGS_HAVE_GC
     )
 
 from pypy.module.cpyext.cparser import CTypeSpace
@@ -419,6 +419,8 @@
     if pto.c_tp_itemsize < base_pto.c_tp_itemsize:
         pto.c_tp_itemsize = base_pto.c_tp_itemsize
 
+    pto.c_tp_flags |= (base_pto.c_tp_flags & Py_TPFLAGS_HAVE_GC)
+
     #/* Setup fast subclass flags */
     if space.issubtype_w(w_obj, space.w_BaseException):
         pto.c_tp_flags |= Py_TPFLAGS_BASE_EXC_SUBCLASS
@@ -432,6 +434,7 @@
         pto.c_tp_flags |= Py_TPFLAGS_UNICODE_SUBCLASS
     elif space.issubtype_w(w_obj, space.w_tuple):
         pto.c_tp_flags |= Py_TPFLAGS_TUPLE_SUBCLASS
+        pto.c_tp_flags |= Py_TPFLAGS_HAVE_GC
     elif space.issubtype_w(w_obj, space.w_list):
         pto.c_tp_flags |= Py_TPFLAGS_LIST_SUBCLASS
     elif space.issubtype_w(w_obj, space.w_dict):
@@ -606,9 +609,9 @@
         pto.c_tp_itemsize = 1
     elif space.is_w(w_type, space.w_tuple):
         pto.c_tp_itemsize = rffi.sizeof(PyObject)
+        pto.c_tp_flags |= Py_TPFLAGS_HAVE_GC
 
     state = space.fromcache(State)
-    pto.c_tp_free = state.C.PyObject_Free
     pto.c_tp_alloc = state.C.PyType_GenericAlloc
     builder = state.builder
     if ((pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE) != 0
@@ -643,6 +646,15 @@
             # strange, but happens (ABCMeta)
             pto.c_tp_dealloc = state.C._PyPy_subtype_dealloc
 
+    # free
+    if space.gettypeobject(w_type.layout.typedef) is w_type:
+        # only for the exact type, like 'space.w_tuple' or 'space.w_list'
+        pto.c_tp_free = typedescr.get_free(space)
+    else:
+        pto.c_tp_free = pto.c_tp_base.c_tp_free
+
+    # TODO: traverse (for tuple)
+
     if builder.cpyext_type_init is not None:
         builder.cpyext_type_init.append((pto, w_type))
     else:

From pypy.commits at gmail.com  Fri May 10 11:09:51 2019
From: pypy.commits at gmail.com (mattip)
Date: Fri, 10 May 2019 08:09:51 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix translation for MSVC which doesn't
 like void* arithmetic
Message-ID: <5cd5943f.1c69fb81.291d8.f8fe@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r96592:d4c7063c42e8
Date: 2019-05-10 06:14 -0700
http://bitbucket.org/pypy/pypy/changeset/d4c7063c42e8/

Log:	fix translation for MSVC which doesn't like void* arithmetic

diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py
--- a/rpython/translator/c/gc.py
+++ b/rpython/translator/c/gc.py
@@ -461,10 +461,10 @@
         raise Exception("gc_pop_roots should be removed by postprocess_graph")
 
     def OP_GC_ENTER_ROOTS_FRAME(self, funcgen, op):
-        return '%s += sizeof(pypy_ss_t);' % (funcgen.gcpol_ss,)
+        return '(char *)(%s) += sizeof(pypy_ss_t);' % (funcgen.gcpol_ss,)
 
     def OP_GC_LEAVE_ROOTS_FRAME(self, funcgen, op):
-        return '%s -= sizeof(pypy_ss_t);' % (funcgen.gcpol_ss,)
+        return 'char *)(%s) -= sizeof(pypy_ss_t);' % (funcgen.gcpol_ss,)
 
     def OP_GC_SAVE_ROOT(self, funcgen, op):
         num = op.args[0].value

From pypy.commits at gmail.com  Fri May 10 19:48:38 2019
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 10 May 2019 16:48:38 -0700 (PDT)
Subject: [pypy-commit] pypy optimizeopt-cleanup: Replace method
 Optimization.getrawptrinfo() with function getrawptrinfo()
Message-ID: <5cd60dd6.1c69fb81.9eb0c.435c@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: optimizeopt-cleanup
Changeset: r96593:34d57de52fa1
Date: 2019-05-11 00:47 +0100
http://bitbucket.org/pypy/pypy/changeset/34d57de52fa1/

Log:	Replace method Optimization.getrawptrinfo() with function
	getrawptrinfo()

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -9,7 +9,6 @@
 from rpython.jit.metainterp.optimize import InvalidLoop
 from .util import get_box_replacement
 
-
 INFO_NULL = 0
 INFO_NONNULL = 1
 INFO_UNKNOWN = 2
@@ -110,7 +109,7 @@
 
     def mark_last_guard(self, optimizer):
         if (optimizer.getlastop() is None or
-            not optimizer.getlastop().is_guard()):
+                not optimizer.getlastop().is_guard()):
             # there can be a really emitted operation that's not a guard
             # e.g. a setfield, ignore those
             return
@@ -468,7 +467,7 @@
         return self.parent.getitem_raw(self.offset+offset, itemsize, descr)
 
     def setitem_raw(self, offset, itemsize, descr, itemop):
-        self.parent.setitem_raw(self.offset+offset, itemsize, descr, itemop)
+        self.parent.setitem_raw(self.offset + offset, itemsize, descr, itemop)
 
     def _force_elements(self, op, optforce, descr):
         if self.parent.is_virtual():
@@ -639,7 +638,7 @@
     def _compute_index(self, index, fielddescr):
         all_fdescrs = fielddescr.get_arraydescr().get_all_fielddescrs()
         if all_fdescrs is None:
-            return 0 # annotation hack
+            return 0  # annotation hack
         one_size = len(all_fdescrs)
         return index * one_size + fielddescr.get_field_descr().get_index()
 
@@ -675,8 +674,7 @@
         for index in range(self.getlength()):
             for fielddescr in fielddescrs:
                 itemop = self._items[i]
-                if (itemop is not None and
-                    not isinstance(itemop, Const)):
+                if (itemop is not None and not isinstance(itemop, Const)):
                     ptrinfo = optimizer.getptrinfo(itemop)
                     if ptrinfo and ptrinfo.is_virtual():
                         ptrinfo.visitor_walk_recursive(itemop, visitor,
@@ -703,7 +701,8 @@
 
     def _get_info(self, descr, optheap):
         ref = self._const.getref_base()
-        if not ref: raise InvalidLoop   # null protection
+        if not ref:
+            raise InvalidLoop   # null protection
         info = optheap.const_infos.get(ref, None)
         if info is None:
             info = StructPtrInfo(descr)
@@ -712,7 +711,8 @@
 
     def _get_array_info(self, descr, optheap):
         ref = self._const.getref_base()
-        if not ref: raise InvalidLoop   # null protection
+        if not ref:
+            raise InvalidLoop   # null protection
         info = optheap.const_infos.get(ref, None)
         if info is None:
             info = ArrayPtrInfo(descr)
@@ -778,8 +778,8 @@
         return self._unpack_str(mode)
 
     def getlenbound(self, mode):
-        from rpython.jit.metainterp.optimizeopt.intutils import ConstIntBound,\
-                IntLowerBound
+        from rpython.jit.metainterp.optimizeopt.intutils import (
+            ConstIntBound, IntLowerBound)
 
         length = self.getstrlen1(mode)
         if length < 0:
@@ -846,3 +846,23 @@
 
     def make_guards(self, op, short, optimizer):
         short.append(ResOperation(rop.GUARD_VALUE, [op, self._const]))
+
+
+def getrawptrinfo(op):
+    from rpython.jit.metainterp.optimizeopt.intutils import IntBound
+    assert op.type == 'i'
+    op = op.get_box_replacement()
+    assert op.type == 'i'
+    if isinstance(op, ConstInt):
+        return ConstPtrInfo(op)
+    fw = op.get_forwarded()
+    if isinstance(fw, IntBound):
+        return None
+    if fw is not None:
+        if isinstance(fw, AbstractRawPtrInfo):
+            return fw
+        fw = RawStructPtrInfo()
+        op.set_forwarded(fw)
+        assert isinstance(fw, AbstractRawPtrInfo)
+        return fw
+    return None
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -10,6 +10,7 @@
     deserialize_optimizer_knowledge)
 from rpython.jit.metainterp.resoperation import (
     rop, AbstractResOp, GuardResOp, OpHelpers)
+from .info import getrawptrinfo
 from rpython.jit.metainterp.optimizeopt import info
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.rlib.objectmodel import specialize, we_are_translated
@@ -137,7 +138,7 @@
 
     def getptrinfo(self, op):
         if op.type == 'i':
-            return self.getrawptrinfo(op)
+            return getrawptrinfo(op)
         elif op.type == 'f':
             return None
         assert op.type == 'r'
@@ -157,24 +158,6 @@
             return True
         return False
 
-    def getrawptrinfo(self, op):
-        assert op.type == 'i'
-        op = get_box_replacement(op)
-        assert op.type == 'i'
-        if isinstance(op, ConstInt):
-            return info.ConstPtrInfo(op)
-        fw = op.get_forwarded()
-        if isinstance(fw, IntBound):
-            return None
-        if fw is not None:
-            if isinstance(fw, info.AbstractRawPtrInfo):
-                return fw
-            fw = info.RawStructPtrInfo()
-            op.set_forwarded(fw)
-            assert isinstance(fw, info.AbstractRawPtrInfo)
-            return fw
-        return None
-
     def replace_op_with(self, op, newopnum, args=None, descr=None):
         return self.optimizer.replace_op_with(op, newopnum, args, descr)
 
@@ -327,7 +310,7 @@
                                                 self.optearlyforce, rec)
             return box
         if box.type == 'i':
-            info = self.getrawptrinfo(box)
+            info = getrawptrinfo(box)
             if info is not None:
                 return info.force_at_the_end_of_preamble(box,
                                             self.optearlyforce, None)
@@ -854,7 +837,7 @@
             opinfo = self.getptrinfo(op)
             return opinfo is not None and opinfo.is_virtual()
         if op.type == 'i':
-            opinfo = self.getrawptrinfo(op)
+            opinfo = getrawptrinfo(op)
             return opinfo is not None and opinfo.is_virtual()
         return False
 
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -5,8 +5,8 @@
 from rpython.jit.metainterp.optimizeopt.optimizer import REMOVED
 from rpython.jit.metainterp.optimizeopt.util import (
     make_dispatcher_method, get_box_replacement)
-
 from rpython.jit.metainterp.optimizeopt.rawbuffer import InvalidRawOperation
+from .info import getrawptrinfo
 from rpython.jit.metainterp.resoperation import rop, ResOperation
 
 
@@ -246,13 +246,13 @@
         self.last_emitted_operation = REMOVED
 
     def do_RAW_FREE(self, op):
-        opinfo = self.getrawptrinfo(op.getarg(1))
+        opinfo = getrawptrinfo(op.getarg(1))
         if opinfo and opinfo.is_virtual():
             return
         return self.emit(op)
 
     def optimize_INT_ADD(self, op):
-        opinfo = self.getrawptrinfo(op.getarg(0))
+        opinfo = getrawptrinfo(op.getarg(0))
         offsetbox = self.get_constant_box(op.getarg(1))
         if opinfo and opinfo.is_virtual() and offsetbox is not None:
             offset = offsetbox.getint()
@@ -315,7 +315,7 @@
         return offset, itemsize, descr
 
     def optimize_GETARRAYITEM_RAW_I(self, op):
-        opinfo = self.getrawptrinfo(op.getarg(0))
+        opinfo = getrawptrinfo(op.getarg(0))
         if opinfo and opinfo.is_virtual():
             indexbox = self.get_constant_box(op.getarg(1))
             if indexbox is not None:
@@ -333,7 +333,7 @@
     optimize_GETARRAYITEM_RAW_F = optimize_GETARRAYITEM_RAW_I
 
     def optimize_SETARRAYITEM_RAW(self, op):
-        opinfo = self.getrawptrinfo(op.getarg(0))
+        opinfo = getrawptrinfo(op.getarg(0))
         if opinfo and opinfo.is_virtual():
             indexbox = self.get_constant_box(op.getarg(1))
             if indexbox is not None:
@@ -355,7 +355,7 @@
         return offset, itemsize, descr
 
     def optimize_RAW_LOAD_I(self, op):
-        opinfo = self.getrawptrinfo(op.getarg(0))
+        opinfo = getrawptrinfo(op.getarg(0))
         if opinfo and opinfo.is_virtual():
             offsetbox = self.get_constant_box(op.getarg(1))
             if offsetbox is not None:
@@ -371,7 +371,7 @@
     optimize_RAW_LOAD_F = optimize_RAW_LOAD_I
 
     def optimize_RAW_STORE(self, op):
-        opinfo = self.getrawptrinfo(op.getarg(0))
+        opinfo = getrawptrinfo(op.getarg(0))
         if opinfo and opinfo.is_virtual():
             offsetbox = self.get_constant_box(op.getarg(1))
             if offsetbox is not None:
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -224,12 +224,13 @@
     def _number_boxes(self, iter, arr, optimizer, numb_state):
         """ Number boxes from one snapshot
         """
+        from rpython.jit.metainterp.optimizeopt.info import getrawptrinfo
         num_boxes = numb_state.num_boxes
         num_virtuals = numb_state.num_virtuals
         liveboxes = numb_state.liveboxes
         for item in arr:
             box = iter.get(rffi.cast(lltype.Signed, item))
-            box = optimizer.get_box_replacement(box)
+            box = box.get_box_replacement()
 
             if isinstance(box, Const):
                 tagged = self.getconst(box)
@@ -241,7 +242,7 @@
                     info = optimizer.getptrinfo(box)
                     is_virtual = (info is not None and info.is_virtual())
                 if box.type == 'i':
-                    info = optimizer.getrawptrinfo(box)
+                    info = getrawptrinfo(box)
                     is_virtual = (info is not None and info.is_virtual())
                 if is_virtual:
                     tagged = tag(num_virtuals, TAGVIRTUAL)
@@ -414,6 +415,7 @@
         return tagbits == TAGVIRTUAL
 
     def finish(self, pending_setfields=[]):
+        from rpython.jit.metainterp.optimizeopt.info import getrawptrinfo
         optimizer = self.optimizer
         # compute the numbering
         storage = self.storage
@@ -442,7 +444,7 @@
                     info = optimizer.getptrinfo(box)
                 else:
                     assert box.type == 'i'
-                    info = optimizer.getrawptrinfo(box)
+                    info = getrawptrinfo(box)
                 assert info.is_virtual()
                 info.visitor_walk_recursive(box, self, optimizer)
 
diff --git a/rpython/jit/metainterp/test/test_resume.py b/rpython/jit/metainterp/test/test_resume.py
--- a/rpython/jit/metainterp/test/test_resume.py
+++ b/rpython/jit/metainterp/test/test_resume.py
@@ -15,7 +15,7 @@
     unpack_numbering, create_numbering)
 from rpython.jit.metainterp.opencoder import Trace
 
-from rpython.jit.metainterp.optimizeopt import info
+from rpython.jit.metainterp.optimizeopt import info, util
 from rpython.jit.metainterp.history import (
     ConstInt, Const, AbstractDescr, ConstPtr, ConstFloat, IntFrontendOp,
     RefFrontendOp, CONST_NULL)
@@ -51,12 +51,8 @@
             op = op.get_forwarded()
         return op
 
-    def getrawptrinfo(self, op):
-        op = self.get_box_replacement(op)
-        return op.get_forwarded()
-
     def getptrinfo(self, op):
-        op = self.get_box_replacement(op)
+        op = util.get_box_replacement(op)
         return op.get_forwarded()
 
 

From pypy.commits at gmail.com  Fri May 10 19:49:10 2019
From: pypy.commits at gmail.com (mattip)
Date: Fri, 10 May 2019 16:49:10 -0700 (PDT)
Subject: [pypy-commit] pypy default: typo
Message-ID: <5cd60df6.1c69fb81.35fc5.aeea@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r96594:cd27c41c26b5
Date: 2019-05-10 16:47 -0700
http://bitbucket.org/pypy/pypy/changeset/cd27c41c26b5/

Log:	typo

diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py
--- a/rpython/translator/c/gc.py
+++ b/rpython/translator/c/gc.py
@@ -464,7 +464,7 @@
         return '(char *)(%s) += sizeof(pypy_ss_t);' % (funcgen.gcpol_ss,)
 
     def OP_GC_LEAVE_ROOTS_FRAME(self, funcgen, op):
-        return 'char *)(%s) -= sizeof(pypy_ss_t);' % (funcgen.gcpol_ss,)
+        return (char *)(%s) -= sizeof(pypy_ss_t);' % (funcgen.gcpol_ss,)
 
     def OP_GC_SAVE_ROOT(self, funcgen, op):
         num = op.args[0].value

From pypy.commits at gmail.com  Sat May 11 13:47:52 2019
From: pypy.commits at gmail.com (andrewjlawrence)
Date: Sat, 11 May 2019 10:47:52 -0700 (PDT)
Subject: [pypy-commit] pypy winmultiprocessing: Tidy up
Message-ID: <5cd70ac8.1c69fb81.f2ae8.a87d@mx.google.com>

Author: andrewjlawrence
Branch: winmultiprocessing
Changeset: r96595:a66d30fad7bc
Date: 2019-05-11 18:45 +0100
http://bitbucket.org/pypy/pypy/changeset/a66d30fad7bc/

Log:	Tidy up

diff --git a/lib_pypy/_overlapped.py b/lib_pypy/_overlapped.py
--- a/lib_pypy/_overlapped.py
+++ b/lib_pypy/_overlapped.py
@@ -173,7 +173,6 @@
         olderr = _kernel32.GetLastError()
         hascompletedio = HasOverlappedIoCompleted(self.overlapped[0])      
         if not hascompletedio and self.type != OverlappedType.TYPE_NOT_STARTED:
-            
             wait = _kernel32.CancelIoEx(self.handle, self.overlapped)
             ret = self.GetOverlappedResult(wait)
             err = _winapi.ERROR_SUCCESS
@@ -186,6 +185,7 @@
                SetFromWindowsErr(err)
         if self.overlapped[0].hEvent != 0:
             _winapi.CloseHandle(self.overlapped[0].hEvent)
+        _winapi.SetLastError(olderr)
 
     @property
     def event(self):
@@ -216,10 +216,6 @@
         else:
             return transferred[0]
 
-    def getbuffer(self):
-        xxx
-        return None
-
     def cancel(self):
         result = True
         if self.type == OverlappedType.TYPE_NOT_STARTED or self.type == OverlappedType.TYPE_WAIT_NAMED_PIPE_AND_CONNECT:
diff --git a/lib_pypy/_pypy_winbase_build.py b/lib_pypy/_pypy_winbase_build.py
--- a/lib_pypy/_pypy_winbase_build.py
+++ b/lib_pypy/_pypy_winbase_build.py
@@ -113,9 +113,11 @@
 BOOL WINAPI CancelIoEx(HANDLE, LPOVERLAPPED);
 BOOL WINAPI CloseHandle(HANDLE);
 DWORD WINAPI GetLastError(VOID);
+void WINAPI SetLastError(DWORD);
 BOOL WINAPI GetOverlappedResult(HANDLE, LPOVERLAPPED, LPDWORD, BOOL);
 HANDLE WINAPI GetCurrentProcess(void);
 HANDLE OpenProcess(DWORD, BOOL, DWORD);
+void ExitProcess(UINT);
 BOOL WINAPI DuplicateHandle(HANDLE, HANDLE, HANDLE, LPHANDLE,
                             DWORD, BOOL, DWORD);
 BOOL WINAPI CreateProcessA(char *, char *, void *,
diff --git a/lib_pypy/_pypy_winbase_cffi.py b/lib_pypy/_pypy_winbase_cffi.py
--- a/lib_pypy/_pypy_winbase_cffi.py
+++ b/lib_pypy/_pypy_winbase_cffi.py
@@ -3,8 +3,8 @@
 
 ffi = _cffi_backend.FFI('_pypy_winbase_cffi',
     _version = 0x2601,
-    _types = b'\x00\x00\x05\x0D\x00\x01\x5D\x03\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x01\x79\x03\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x01\x53\x03\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x03\x00\x01\x4C\x03\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x01\x5D\x03\x00\x00\x2B\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x2B\x11\x00\x00\x2B\x11\x00\x01\x58\x03\x00\x01\x4D\x03\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x03\x00\x00\x37\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x37\x11\x00\x00\x15\x11\x00\x01\x43\x03\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x26\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x26\x11\x00\x00\x25\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x26\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x19\x11\x00\x00\x07\x01\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x26\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x01\x59\x03\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x25\x11\x00\x00\x26\x11\x00\x01\x3E\x03\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x62\x11\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x0A\x01\x00\x00\x26\x11\x00\x00\x67\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x25\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x25\x11\x00\x00\x25\x03\x00\x00\x26\x03\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x25\x11\x00\x00\x25\x11\x00\x00\x25\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x26\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x26\x11\x00\x00\x67\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x25\x11\x00\x00\x25\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x26\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x37\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x01\x79\x03\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x26\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\xEE\x03\x00\x00\x07\x01\x00\x01\x5C\x03\x00\x00\x19\x11\x00\x00\x05\x03\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\xBF\x11\x00\x00\xBF\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\xBF\x11\x00\x00\xBF\x11\x00\x00\x33\x11\x00\x00\x34\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x74\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x02\x0D\x00\x00\x0A\x01\x00\x00\x37\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x02\x0D\x00\x00\x15\x11\x00\x00\xBF\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x02\x0D\x00\x00\x02\x0F\x00\x00\xE9\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\xE9\x0D\x00\x00\x00\x0F\x00\x00\xE9\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x01\x52\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x01\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\xFD\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xEE\x03\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x01\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xFD\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x01\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xFD\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\xBF\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xFD\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x01\x06\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xFD\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x01\x79\x0D\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x00\x0F\x00\x01\x79\x0D\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x0C\x09\x00\x01\x49\x03\x00\x00\x13\x09\x00\x01\x4B\x03\x00\x00\x14\x09\x00\x00\x0D\x09\x00\x00\x09\x09\x00\x01\x4F\x03\x00\x00\x0E\x09\x00\x01\x51\x03\x00\x00\x0A\x09\x00\x00\x0F\x09\x00\x00\x15\x09\x00\x01\x57\x03\x00\x01\x56\x03\x00\x00\x17\x09\x00\x00\x16\x09\x00\x00\x0B\x09\x00\x00\x10\x09\x00\x01\x5B\x03\x00\x00\x11\x09\x00\x00\x12\x09\x00\x00\x02\x01\x00\x01\x5D\x05\x00\x00\x00\x0E\x00\x01\x5D\x05\x00\x00\x00\x08\x00\x00\x51\x03\x00\x00\x57\x03\x00\x00\xA4\x03\x00\x00\x05\x01\x00\x00\x01\x09\x00\x00\x04\x09\x00\x00\x07\x09\x00\x00\x08\x09\x00\x00\x00\x09\x00\x00\x02\x09\x00\x00\x03\x09\x00\x00\x05\x09\x00\x00\x06\x09\x00\x01\x70\x03\x00\x00\x04\x01\x00\x01\x70\x05\x00\x00\x00\x10\x00\x01\x70\x05\x00\x00\x00\x08\x00\x00\x02\x05\x00\x00\x00\x07\x00\x00\xE9\x05\x00\x00\x00\x08\x00\x00\x00\x01\x00\x00\xEE\x05\x00\x00\x01\x00',
-    _globals = (b'\x00\x00\x44\x23CancelIo',0,b'\x00\x00\x47\x23CancelIoEx',0,b'\x00\x00\x44\x23CloseHandle',0,b'\x00\x00\x47\x23ConnectNamedPipe',0,b'\x00\x00\xFC\x23CreateEventA',0,b'\x00\x01\x02\x23CreateEventW',0,b'\x00\x01\x08\x23CreateFileA',0,b'\x00\x01\x35\x23CreateFileW',0,b'\x00\x01\x23\x23CreateIoCompletionPort',0,b'\x00\x01\x11\x23CreateNamedPipeA',0,b'\x00\x01\x2B\x23CreateNamedPipeW',0,b'\x00\x00\x36\x23CreatePipe',0,b'\x00\x00\x2A\x23CreateProcessA',0,b'\x00\x00\xC5\x23CreateProcessW',0,b'\x00\x00\xAE\x23DuplicateHandle',0,b'\x00\x01\x29\x23GetCurrentProcess',0,b'\x00\x00\x76\x23GetExitCodeProcess',0,b'\x00\x00\xE6\x23GetLastError',0,b'\x00\x00\xE1\x23GetModuleFileNameW',0,b'\x00\x00\x4B\x23GetOverlappedResult',0,b'\x00\x00\xFA\x23GetProcessHeap',0,b'\x00\x00\x7A\x23GetQueuedCompletionStatus',0,b'\x00\x01\x20\x23GetStdHandle',0,b'\x00\x00\xE6\x23GetVersion',0,b'\x00\x00\xF5\x23HeapAlloc',0,b'\x00\x00\x1C\x23HeapFree',0,b'\xFF\xFF\xFF\x1FMAX_PROTOCOL_CHAIN',7,b'\x00\x00\xF0\x23OpenProcess',0,b'\x00\x00\x9C\x23PeekNamedPipe',0,b'\x00\x00\x87\x23PostQueuedCompletionStatus',0,b'\x00\x00\x21\x23ReadFile',0,b'\x00\x00\x3C\x23RegisterWaitForSingleObject',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\xD4\x23SetErrorMode',0,b'\x00\x00\x44\x23SetEvent',0,b'\x00\x00\x81\x23SetNamedPipeHandleState',0,b'\x00\x00\x72\x23TerminateProcess',0,b'\x00\x00\x44\x23UnregisterWait',0,b'\x00\x00\x98\x23UnregisterWaitEx',0,b'\x00\x00\x8D\x23WSAIoctl',0,b'\xFF\xFF\xFF\x1FWSAPROTOCOL_LEN',255,b'\x00\x00\x60\x23WSARecv',0,b'\x00\x00\x69\x23WSASend',0,b'\x00\x00\xBE\x23WSAStringToAddressW',0,b'\xFF\xFF\xFF\x1FWT_EXECUTEINWAITTHREAD',4,b'\xFF\xFF\xFF\x1FWT_EXECUTEONLYONCE',8,b'\x00\x00\xD7\x23WaitForMultipleObjects',0,b'\x00\x00\xDD\x23WaitForSingleObject',0,b'\x00\x00\x00\x23WaitNamedPipeA',0,b'\x00\x00\xB7\x23WriteFile',0,b'\x00\x00\xD1\x23_get_osfhandle',0,b'\x00\x00\x28\x23_getch',0,b'\x00\x00\x28\x23_getche',0,b'\x00\x00\xEB\x23_getwch',0,b'\x00\x00\xEB\x23_getwche',0,b'\x00\x00\x28\x23_kbhit',0,b'\x00\x00\x0B\x23_locking',0,b'\x00\x00\x10\x23_open_osfhandle',0,b'\x00\x00\x04\x23_putch',0,b'\x00\x00\xED\x23_putwch',0,b'\x00\x00\x07\x23_setmode',0,b'\x00\x00\x04\x23_ungetch',0,b'\x00\x00\xE8\x23_ungetwch',0,b'\x00\x00\x17\x23bind',0,b'\x00\x00\x14\x23closesocket',0,b'\x00\x00\xE8\x23htons',0,b'\x00\x01\x1B\x23socket',0),
-    _struct_unions = ((b'\x00\x00\x01\x6A\x00\x00\x00\x03$1',b'\x00\x01\x66\x11DUMMYSTRUCTNAME',b'\x00\x00\x15\x11Pointer'),(b'\x00\x00\x01\x66\x00\x00\x00\x02$2',b'\x00\x00\x02\x11Offset',b'\x00\x00\x02\x11OffsetHigh'),(b'\x00\x00\x01\x6B\x00\x00\x00\x03$3',b'\x00\x01\x71\x11Byte',b'\x00\x01\x77\x11Word'),(b'\x00\x00\x01\x6C\x00\x00\x00\x01$4',b'\x00\x01\x67\x11',b'\x00\x00\x02\x11Value'),(b'\x00\x00\x01\x67\x00\x00\x00\x02$5',b'\x00\x00\x02\x13\x00\x00\x00\x1CZone',b'\x00\x00\x02\x13\x00\x00\x00\x04Level'),(b'\x00\x00\x01\x6D\x00\x00\x00\x03$6',b'\x00\x00\x02\x11sin6_scope_id',b'\x00\x01\x51\x11sin6_scope_struct'),(b'\x00\x00\x01\x6E\x00\x00\x00\x03$7',b'\x00\x01\x68\x11S_un_b',b'\x00\x01\x69\x11S_un_w',b'\x00\x00\x02\x11S_addr'),(b'\x00\x00\x01\x68\x00\x00\x00\x02$8',b'\x00\x01\x70\x11s_b1',b'\x00\x01\x70\x11s_b2',b'\x00\x01\x70\x11s_b3',b'\x00\x01\x70\x11s_b4'),(b'\x00\x00\x01\x69\x00\x00\x00\x02$9',b'\x00\x00\xE9\x11s_w1',b'\x00\x00\xE9\x11s_w2'),(b'\x00\x00\x01\x4D\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x02\x11dwProcessId',b'\x00\x00\x02\x11dwThreadId'),(b'\x00\x00\x01\x51\x00\x00\x00\x00$SCOPE_ID',b'\x00\x01\x6C\x11'),(b'\x00\x00\x01\x58\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x02\x11cb',b'\x00\x00\x2B\x11lpReserved',b'\x00\x00\x2B\x11lpDesktop',b'\x00\x00\x2B\x11lpTitle',b'\x00\x00\x02\x11dwX',b'\x00\x00\x02\x11dwY',b'\x00\x00\x02\x11dwXSize',b'\x00\x00\x02\x11dwYSize',b'\x00\x00\x02\x11dwXCountChars',b'\x00\x00\x02\x11dwYCountChars',b'\x00\x00\x02\x11dwFillAttribute',b'\x00\x00\x02\x11dwFlags',b'\x00\x00\xE9\x11wShowWindow',b'\x00\x00\xE9\x11cbReserved2',b'\x00\x01\x6F\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError'),(b'\x00\x00\x01\x47\x00\x00\x00\x02_GUID',b'\x00\x00\x02\x11Data1',b'\x00\x00\xE9\x11Data2',b'\x00\x00\xE9\x11Data3',b'\x00\x01\x73\x11Data4'),(b'\x00\x00\x01\x4C\x00\x00\x00\x02_OVERLAPPED',b'\x00\x00\x02\x11Internal',b'\x00\x00\x02\x11InternalHigh',b'\x00\x01\x6A\x11DUMMYUNIONNAME',b'\x00\x00\x15\x11hEvent'),(b'\x00\x00\x01\x4F\x00\x00\x00\x02_PostCallbackData',b'\x00\x00\x15\x11hCompletionPort',b'\x00\x00\x26\x11Overlapped'),(b'\x00\x00\x01\x52\x00\x00\x00\x02_SECURITY_ATTRIBUTES',b'\x00\x00\x02\x11nLength',b'\x00\x00\x15\x11lpSecurityDescriptor',b'\x00\x00\x05\x11bInheritHandle'),(b'\x00\x00\x01\x59\x00\x00\x00\x02_WSABUF',b'\x00\x00\x02\x11len',b'\x00\x00\x2B\x11buf'),(b'\x00\x00\x01\x5B\x00\x00\x00\x02_WSAPROTOCOLCHAIN',b'\x00\x00\x05\x11ChainLen',b'\x00\x01\x75\x11ChainEntries'),(b'\x00\x00\x01\x5C\x00\x00\x00\x02_WSAPROTOCOL_INFOW',b'\x00\x00\x02\x11dwServiceFlags1',b'\x00\x00\x02\x11dwServiceFlags2',b'\x00\x00\x02\x11dwServiceFlags3',b'\x00\x00\x02\x11dwServiceFlags4',b'\x00\x00\x02\x11dwProviderFlags',b'\x00\x01\x47\x11ProviderId',b'\x00\x00\x02\x11dwCatalogEntryId',b'\x00\x01\x5B\x11ProtocolChain',b'\x00\x00\x05\x11iVersion',b'\x00\x00\x05\x11iAddressFamily',b'\x00\x00\x05\x11iMaxSockAddr',b'\x00\x00\x05\x11iMinSockAddr',b'\x00\x00\x05\x11iSocketType',b'\x00\x00\x05\x11iProtocol',b'\x00\x00\x05\x11iProtocolMaxOffset',b'\x00\x00\x05\x11iNetworkByteOrder',b'\x00\x00\x05\x11iSecurityScheme',b'\x00\x00\x02\x11dwMessageSize',b'\x00\x00\x02\x11dwProviderReserved',b'\x00\x01\x7A\x11szProtocol'),(b'\x00\x00\x01\x49\x00\x00\x00\x02in6_addr',b'\x00\x01\x6B\x11u'),(b'\x00\x00\x01\x4B\x00\x00\x00\x02in_addr',b'\x00\x01\x6E\x11S_un'),(b'\x00\x00\x01\x53\x00\x00\x00\x02sockaddr',b'\x00\x00\xE9\x11sa_family',b'\x00\x01\x5E\x11sa_data'),(b'\x00\x00\x01\x57\x00\x00\x00\x02sockaddr_in',b'\x00\x01\x65\x11sin_family',b'\x00\x00\xE9\x11sin_port',b'\x00\x01\x4B\x11sin_addr',b'\x00\x01\x60\x11sin_zero'),(b'\x00\x00\x01\x56\x00\x00\x00\x00sockaddr_in6',b'\x00\x00\xE9\x11sin6_family',b'\x00\x00\xE9\x11sin6_port',b'\x00\x00\x02\x11sin6_flowinfo',b'\x00\x01\x49\x11sin6_addr',b'\x00\x01\x6D\x11')),
-    _typenames = (b'\x00\x00\x00\xE9ADDRESS_FAMILY',b'\x00\x00\x01\x64AcceptExPtr',b'\x00\x00\x01\x63ConnectExPtr',b'\x00\x00\x01\x62DisconnectExPtr',b'\x00\x00\x01\x47GUID',b'\x00\x00\x01\x49IN6_ADDR',b'\x00\x00\x01\x4BINADDR',b'\x00\x00\x01\x62LPFN_DISCONNECTEX',b'\x00\x00\x01\x48LPIN6_ADDR',b'\x00\x00\x00\x26LPOVERLAPPED',b'\x00\x00\x00\x67LPOVERLAPPED_COMPLETION_ROUTINE',b'\x00\x00\x00\x34LPPROCESS_INFORMATION',b'\x00\x00\x01\x4ELPPostCallbackData',b'\x00\x00\x00\xFDLPSECURITY_ATTRIBUTES',b'\x00\x00\x00\x19LPSOCKADDR',b'\x00\x00\x01\x54LPSOCKADDR_IN',b'\x00\x00\x01\x55LPSOCKADDR_IN6_LH',b'\x00\x00\x00\x33LPSTARTUPINFO',b'\x00\x00\x00\x62LPWSABUF',b'\x00\x00\x01\x5ALPWSAPROTOCOLCHAIN',b'\x00\x00\x00\xC1LPWSAPROTOCOL_INFOW',b'\x00\x00\x01\x4COVERLAPPED',b'\x00\x00\x01\x48PIN6_ADDR',b'\x00\x00\x01\x4APINADDR',b'\x00\x00\x01\x4DPROCESS_INFORMATION',b'\x00\x00\x01\x50PSCOPE_ID',b'\x00\x00\x00\xFDPSECURITY_ATTRIBUTES',b'\x00\x00\x00\x19PSOCKADDR',b'\x00\x00\x01\x54PSOCKADDR_IN',b'\x00\x00\x01\x55PSOCKADDR_IN6_LH',b'\x00\x00\x01\x4FPostCallbackData',b'\x00\x00\x01\x51SCOPE_ID',b'\x00\x00\x01\x52SECURITY_ATTRIBUTES',b'\x00\x00\x01\x53SOCKADDR',b'\x00\x00\x01\x57SOCKADDR_IN',b'\x00\x00\x01\x56SOCKADDR_IN6_LH',b'\x00\x00\x00\x15SOCKET',b'\x00\x00\x01\x58STARTUPINFO',b'\x00\x00\x00\x3FWAITORTIMERCALLBACK',b'\x00\x00\x01\x59WSABUF',b'\x00\x00\x01\x5BWSAPROTOCOLCHAIN',b'\x00\x00\x01\x5CWSAPROTOCOL_INFOW',b'\x00\x00\x00\xE9wint_t'),
+    _types = b'\x00\x00\x05\x0D\x00\x01\x63\x03\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x01\x7F\x03\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x01\x59\x03\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x03\x00\x01\x52\x03\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x01\x63\x03\x00\x00\x2B\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x2B\x11\x00\x00\x2B\x11\x00\x01\x5E\x03\x00\x01\x53\x03\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x03\x00\x00\x37\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x37\x11\x00\x00\x15\x11\x00\x01\x49\x03\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x26\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x26\x11\x00\x00\x25\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x26\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x19\x11\x00\x00\x07\x01\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x26\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x01\x5F\x03\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x25\x11\x00\x00\x26\x11\x00\x01\x41\x03\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x62\x11\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x0A\x01\x00\x00\x26\x11\x00\x00\x67\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x25\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x25\x11\x00\x00\x25\x03\x00\x00\x26\x03\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x25\x11\x00\x00\x25\x11\x00\x00\x25\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x26\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x26\x11\x00\x00\x67\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x25\x11\x00\x00\x25\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x26\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x37\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x01\x7F\x03\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x26\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\xEE\x03\x00\x00\x07\x01\x00\x01\x62\x03\x00\x00\x19\x11\x00\x00\x05\x03\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\xBF\x11\x00\x00\xBF\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\xBF\x11\x00\x00\xBF\x11\x00\x00\x33\x11\x00\x00\x34\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x74\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x02\x0D\x00\x00\x0A\x01\x00\x00\x37\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x02\x0D\x00\x00\x15\x11\x00\x00\xBF\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x02\x0D\x00\x00\x02\x0F\x00\x00\xE9\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\xE9\x0D\x00\x00\x00\x0F\x00\x00\xE9\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x01\x58\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x01\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\xFD\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xEE\x03\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x01\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xFD\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x01\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xFD\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\xBF\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xFD\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x01\x06\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xFD\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x01\x7F\x0D\x00\x00\x08\x01\x00\x00\x00\x0F\x00\x01\x7F\x0D\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x00\x0F\x00\x01\x7F\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x01\x7F\x0D\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x0C\x09\x00\x01\x4F\x03\x00\x00\x13\x09\x00\x01\x51\x03\x00\x00\x14\x09\x00\x00\x0D\x09\x00\x00\x09\x09\x00\x01\x55\x03\x00\x00\x0E\x09\x00\x01\x57\x03\x00\x00\x0A\x09\x00\x00\x0F\x09\x00\x00\x15\x09\x00\x01\x5D\x03\x00\x01\x5C\x03\x00\x00\x17\x09\x00\x00\x16\x09\x00\x00\x0B\x09\x00\x00\x10\x09\x00\x01\x61\x03\x00\x00\x11\x09\x00\x00\x12\x09\x00\x00\x02\x01\x00\x01\x63\x05\x00\x00\x00\x0E\x00\x01\x63\x05\x00\x00\x00\x08\x00\x00\x51\x03\x00\x00\x57\x03\x00\x00\xA4\x03\x00\x00\x05\x01\x00\x00\x01\x09\x00\x00\x04\x09\x00\x00\x07\x09\x00\x00\x08\x09\x00\x00\x00\x09\x00\x00\x02\x09\x00\x00\x03\x09\x00\x00\x05\x09\x00\x00\x06\x09\x00\x01\x76\x03\x00\x00\x04\x01\x00\x01\x76\x05\x00\x00\x00\x10\x00\x01\x76\x05\x00\x00\x00\x08\x00\x00\x02\x05\x00\x00\x00\x07\x00\x00\xE9\x05\x00\x00\x00\x08\x00\x00\x00\x01\x00\x00\xEE\x05\x00\x00\x01\x00',
+    _globals = (b'\x00\x00\x44\x23CancelIo',0,b'\x00\x00\x47\x23CancelIoEx',0,b'\x00\x00\x44\x23CloseHandle',0,b'\x00\x00\x47\x23ConnectNamedPipe',0,b'\x00\x00\xFC\x23CreateEventA',0,b'\x00\x01\x02\x23CreateEventW',0,b'\x00\x01\x08\x23CreateFileA',0,b'\x00\x01\x35\x23CreateFileW',0,b'\x00\x01\x23\x23CreateIoCompletionPort',0,b'\x00\x01\x11\x23CreateNamedPipeA',0,b'\x00\x01\x2B\x23CreateNamedPipeW',0,b'\x00\x00\x36\x23CreatePipe',0,b'\x00\x00\x2A\x23CreateProcessA',0,b'\x00\x00\xC5\x23CreateProcessW',0,b'\x00\x00\xAE\x23DuplicateHandle',0,b'\x00\x01\x3E\x23ExitProcess',0,b'\x00\x01\x29\x23GetCurrentProcess',0,b'\x00\x00\x76\x23GetExitCodeProcess',0,b'\x00\x00\xE6\x23GetLastError',0,b'\x00\x00\xE1\x23GetModuleFileNameW',0,b'\x00\x00\x4B\x23GetOverlappedResult',0,b'\x00\x00\xFA\x23GetProcessHeap',0,b'\x00\x00\x7A\x23GetQueuedCompletionStatus',0,b'\x00\x01\x20\x23GetStdHandle',0,b'\x00\x00\xE6\x23GetVersion',0,b'\x00\x00\xF5\x23HeapAlloc',0,b'\x00\x00\x1C\x23HeapFree',0,b'\xFF\xFF\xFF\x1FMAX_PROTOCOL_CHAIN',7,b'\x00\x00\xF0\x23OpenProcess',0,b'\x00\x00\x9C\x23PeekNamedPipe',0,b'\x00\x00\x87\x23PostQueuedCompletionStatus',0,b'\x00\x00\x21\x23ReadFile',0,b'\x00\x00\x3C\x23RegisterWaitForSingleObject',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\xD4\x23SetErrorMode',0,b'\x00\x00\x44\x23SetEvent',0,b'\x00\x01\x46\x23SetLastError',0,b'\x00\x00\x81\x23SetNamedPipeHandleState',0,b'\x00\x00\x72\x23TerminateProcess',0,b'\x00\x00\x44\x23UnregisterWait',0,b'\x00\x00\x98\x23UnregisterWaitEx',0,b'\x00\x00\x8D\x23WSAIoctl',0,b'\xFF\xFF\xFF\x1FWSAPROTOCOL_LEN',255,b'\x00\x00\x60\x23WSARecv',0,b'\x00\x00\x69\x23WSASend',0,b'\x00\x00\xBE\x23WSAStringToAddressW',0,b'\xFF\xFF\xFF\x1FWT_EXECUTEINWAITTHREAD',4,b'\xFF\xFF\xFF\x1FWT_EXECUTEONLYONCE',8,b'\x00\x00\xD7\x23WaitForMultipleObjects',0,b'\x00\x00\xDD\x23WaitForSingleObject',0,b'\x00\x00\x00\x23WaitNamedPipeA',0,b'\x00\x00\xB7\x23WriteFile',0,b'\x00\x00\xD1\x23_get_osfhandle',0,b'\x00\x00\x28\x23_getch',0,b'\x00\x00\x28\x23_getche',0,b'\x00\x00\xEB\x23_getwch',0,b'\x00\x00\xEB\x23_getwche',0,b'\x00\x00\x28\x23_kbhit',0,b'\x00\x00\x0B\x23_locking',0,b'\x00\x00\x10\x23_open_osfhandle',0,b'\x00\x00\x04\x23_putch',0,b'\x00\x00\xED\x23_putwch',0,b'\x00\x00\x07\x23_setmode',0,b'\x00\x00\x04\x23_ungetch',0,b'\x00\x00\xE8\x23_ungetwch',0,b'\x00\x00\x17\x23bind',0,b'\x00\x00\x14\x23closesocket',0,b'\x00\x00\xE8\x23htons',0,b'\x00\x01\x1B\x23socket',0),
+    _struct_unions = ((b'\x00\x00\x01\x70\x00\x00\x00\x03$1',b'\x00\x01\x6C\x11DUMMYSTRUCTNAME',b'\x00\x00\x15\x11Pointer'),(b'\x00\x00\x01\x6C\x00\x00\x00\x02$2',b'\x00\x00\x02\x11Offset',b'\x00\x00\x02\x11OffsetHigh'),(b'\x00\x00\x01\x71\x00\x00\x00\x03$3',b'\x00\x01\x77\x11Byte',b'\x00\x01\x7D\x11Word'),(b'\x00\x00\x01\x72\x00\x00\x00\x01$4',b'\x00\x01\x6D\x11',b'\x00\x00\x02\x11Value'),(b'\x00\x00\x01\x6D\x00\x00\x00\x02$5',b'\x00\x00\x02\x13\x00\x00\x00\x1CZone',b'\x00\x00\x02\x13\x00\x00\x00\x04Level'),(b'\x00\x00\x01\x73\x00\x00\x00\x03$6',b'\x00\x00\x02\x11sin6_scope_id',b'\x00\x01\x57\x11sin6_scope_struct'),(b'\x00\x00\x01\x74\x00\x00\x00\x03$7',b'\x00\x01\x6E\x11S_un_b',b'\x00\x01\x6F\x11S_un_w',b'\x00\x00\x02\x11S_addr'),(b'\x00\x00\x01\x6E\x00\x00\x00\x02$8',b'\x00\x01\x76\x11s_b1',b'\x00\x01\x76\x11s_b2',b'\x00\x01\x76\x11s_b3',b'\x00\x01\x76\x11s_b4'),(b'\x00\x00\x01\x6F\x00\x00\x00\x02$9',b'\x00\x00\xE9\x11s_w1',b'\x00\x00\xE9\x11s_w2'),(b'\x00\x00\x01\x53\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x02\x11dwProcessId',b'\x00\x00\x02\x11dwThreadId'),(b'\x00\x00\x01\x57\x00\x00\x00\x00$SCOPE_ID',b'\x00\x01\x72\x11'),(b'\x00\x00\x01\x5E\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x02\x11cb',b'\x00\x00\x2B\x11lpReserved',b'\x00\x00\x2B\x11lpDesktop',b'\x00\x00\x2B\x11lpTitle',b'\x00\x00\x02\x11dwX',b'\x00\x00\x02\x11dwY',b'\x00\x00\x02\x11dwXSize',b'\x00\x00\x02\x11dwYSize',b'\x00\x00\x02\x11dwXCountChars',b'\x00\x00\x02\x11dwYCountChars',b'\x00\x00\x02\x11dwFillAttribute',b'\x00\x00\x02\x11dwFlags',b'\x00\x00\xE9\x11wShowWindow',b'\x00\x00\xE9\x11cbReserved2',b'\x00\x01\x75\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError'),(b'\x00\x00\x01\x4D\x00\x00\x00\x02_GUID',b'\x00\x00\x02\x11Data1',b'\x00\x00\xE9\x11Data2',b'\x00\x00\xE9\x11Data3',b'\x00\x01\x79\x11Data4'),(b'\x00\x00\x01\x52\x00\x00\x00\x02_OVERLAPPED',b'\x00\x00\x02\x11Internal',b'\x00\x00\x02\x11InternalHigh',b'\x00\x01\x70\x11DUMMYUNIONNAME',b'\x00\x00\x15\x11hEvent'),(b'\x00\x00\x01\x55\x00\x00\x00\x02_PostCallbackData',b'\x00\x00\x15\x11hCompletionPort',b'\x00\x00\x26\x11Overlapped'),(b'\x00\x00\x01\x58\x00\x00\x00\x02_SECURITY_ATTRIBUTES',b'\x00\x00\x02\x11nLength',b'\x00\x00\x15\x11lpSecurityDescriptor',b'\x00\x00\x05\x11bInheritHandle'),(b'\x00\x00\x01\x5F\x00\x00\x00\x02_WSABUF',b'\x00\x00\x02\x11len',b'\x00\x00\x2B\x11buf'),(b'\x00\x00\x01\x61\x00\x00\x00\x02_WSAPROTOCOLCHAIN',b'\x00\x00\x05\x11ChainLen',b'\x00\x01\x7B\x11ChainEntries'),(b'\x00\x00\x01\x62\x00\x00\x00\x02_WSAPROTOCOL_INFOW',b'\x00\x00\x02\x11dwServiceFlags1',b'\x00\x00\x02\x11dwServiceFlags2',b'\x00\x00\x02\x11dwServiceFlags3',b'\x00\x00\x02\x11dwServiceFlags4',b'\x00\x00\x02\x11dwProviderFlags',b'\x00\x01\x4D\x11ProviderId',b'\x00\x00\x02\x11dwCatalogEntryId',b'\x00\x01\x61\x11ProtocolChain',b'\x00\x00\x05\x11iVersion',b'\x00\x00\x05\x11iAddressFamily',b'\x00\x00\x05\x11iMaxSockAddr',b'\x00\x00\x05\x11iMinSockAddr',b'\x00\x00\x05\x11iSocketType',b'\x00\x00\x05\x11iProtocol',b'\x00\x00\x05\x11iProtocolMaxOffset',b'\x00\x00\x05\x11iNetworkByteOrder',b'\x00\x00\x05\x11iSecurityScheme',b'\x00\x00\x02\x11dwMessageSize',b'\x00\x00\x02\x11dwProviderReserved',b'\x00\x01\x80\x11szProtocol'),(b'\x00\x00\x01\x4F\x00\x00\x00\x02in6_addr',b'\x00\x01\x71\x11u'),(b'\x00\x00\x01\x51\x00\x00\x00\x02in_addr',b'\x00\x01\x74\x11S_un'),(b'\x00\x00\x01\x59\x00\x00\x00\x02sockaddr',b'\x00\x00\xE9\x11sa_family',b'\x00\x01\x64\x11sa_data'),(b'\x00\x00\x01\x5D\x00\x00\x00\x02sockaddr_in',b'\x00\x01\x6B\x11sin_family',b'\x00\x00\xE9\x11sin_port',b'\x00\x01\x51\x11sin_addr',b'\x00\x01\x66\x11sin_zero'),(b'\x00\x00\x01\x5C\x00\x00\x00\x00sockaddr_in6',b'\x00\x00\xE9\x11sin6_family',b'\x00\x00\xE9\x11sin6_port',b'\x00\x00\x02\x11sin6_flowinfo',b'\x00\x01\x4F\x11sin6_addr',b'\x00\x01\x73\x11')),
+    _typenames = (b'\x00\x00\x00\xE9ADDRESS_FAMILY',b'\x00\x00\x01\x6AAcceptExPtr',b'\x00\x00\x01\x69ConnectExPtr',b'\x00\x00\x01\x68DisconnectExPtr',b'\x00\x00\x01\x4DGUID',b'\x00\x00\x01\x4FIN6_ADDR',b'\x00\x00\x01\x51INADDR',b'\x00\x00\x01\x68LPFN_DISCONNECTEX',b'\x00\x00\x01\x4ELPIN6_ADDR',b'\x00\x00\x00\x26LPOVERLAPPED',b'\x00\x00\x00\x67LPOVERLAPPED_COMPLETION_ROUTINE',b'\x00\x00\x00\x34LPPROCESS_INFORMATION',b'\x00\x00\x01\x54LPPostCallbackData',b'\x00\x00\x00\xFDLPSECURITY_ATTRIBUTES',b'\x00\x00\x00\x19LPSOCKADDR',b'\x00\x00\x01\x5ALPSOCKADDR_IN',b'\x00\x00\x01\x5BLPSOCKADDR_IN6_LH',b'\x00\x00\x00\x33LPSTARTUPINFO',b'\x00\x00\x00\x62LPWSABUF',b'\x00\x00\x01\x60LPWSAPROTOCOLCHAIN',b'\x00\x00\x00\xC1LPWSAPROTOCOL_INFOW',b'\x00\x00\x01\x52OVERLAPPED',b'\x00\x00\x01\x4EPIN6_ADDR',b'\x00\x00\x01\x50PINADDR',b'\x00\x00\x01\x53PROCESS_INFORMATION',b'\x00\x00\x01\x56PSCOPE_ID',b'\x00\x00\x00\xFDPSECURITY_ATTRIBUTES',b'\x00\x00\x00\x19PSOCKADDR',b'\x00\x00\x01\x5APSOCKADDR_IN',b'\x00\x00\x01\x5BPSOCKADDR_IN6_LH',b'\x00\x00\x01\x55PostCallbackData',b'\x00\x00\x01\x57SCOPE_ID',b'\x00\x00\x01\x58SECURITY_ATTRIBUTES',b'\x00\x00\x01\x59SOCKADDR',b'\x00\x00\x01\x5DSOCKADDR_IN',b'\x00\x00\x01\x5CSOCKADDR_IN6_LH',b'\x00\x00\x00\x15SOCKET',b'\x00\x00\x01\x5ESTARTUPINFO',b'\x00\x00\x00\x3FWAITORTIMERCALLBACK',b'\x00\x00\x01\x5FWSABUF',b'\x00\x00\x01\x61WSAPROTOCOLCHAIN',b'\x00\x00\x01\x62WSAPROTOCOL_INFOW',b'\x00\x00\x00\xE9wint_t'),
 )
diff --git a/lib_pypy/_winapi.py b/lib_pypy/_winapi.py
--- a/lib_pypy/_winapi.py
+++ b/lib_pypy/_winapi.py
@@ -5,6 +5,7 @@
 """
 
 import sys
+
 if sys.platform != 'win32':
     raise ImportError("The '_winapi' module is only available on Windows")
 
@@ -16,6 +17,8 @@
 GetVersion = _kernel32.GetVersion
 NULL = _ffi.NULL
 
+def SetLastError(errno):
+    return _kernel32.SetLastError(errno)
 
 def GetLastError():
     return _kernel32.GetLastError()
@@ -55,19 +58,19 @@
     res = _kernel32.CreatePipe(handles, handles + 1, NULL, size)
 
     if not res:
-        SetFromWindowsErr(0)
+        SetFromWindowsErr(GetLastError())
 
     return _handle2int(handles[0]), _handle2int(handles[1])
 
 def CreateNamedPipe(*args):
     handle = _kernel32.CreateNamedPipeW(*args)
-    if handle == INVALID_HANDLE_VALUE:
+    if handle == _INVALID_HANDLE_VALUE:
         SetFromWindowsErr(0)
     return _handle2int(handle)
 
 def CreateFile(*args):
     handle = _kernel32.CreateFileW(*args)
-    if handle == INVALID_HANDLE_VALUE:
+    if handle == _INVALID_HANDLE_VALUE:
         SetFromWindowsErr(0)
     return _handle2int(handle)
 
@@ -88,7 +91,7 @@
 class Overlapped(object):
     def __init__(self, handle):
         self.overlapped = _ffi.new('OVERLAPPED[1]')
-        self.handle = handle
+        self.handle = _handle2int(handle)
         self.readbuffer = None
         self.pending = 0
         self.completed = 0
@@ -100,14 +103,18 @@
         # do this somehow else
         err = _kernel32.GetLastError()
         bytes = _ffi.new('DWORD[1]')
-        o = self.overlapped[0]
         if self.pending:
-            if _kernel32.CancelIoEx(_int2handle(self.handle), self.overlapped) and \
-                _kernel32.GetOverlappedResult(_int2handle(self.handle), self.overlapped, bytes, True):
+            result = _kernel32.CancelIoEx(_int2handle(self.handle), self.overlapped)
+            if result: 
+                _kernel32.GetOverlappedResult(_int2handle(self.handle), self.overlapped, bytes, True)
                 # The operation is no longer pending, nothing to do
-                pass
+                
             #else:
+                # We need to raise a warning here and not crash pypy
                 #raise RuntimeError('deleting an overlapped struct with a pending operation not supported')
+        CloseHandle(_int2handle(self.overlapped[0].hEvent))
+        _kernel32.SetLastError(err)
+        err = _kernel32.GetLastError()
 
     @property
     def event(self):
@@ -115,7 +122,8 @@
 
     def GetOverlappedResult(self, wait):
         transferred = _ffi.new('DWORD[1]', [0])
-        res = _kernel32.GetOverlappedResult(_int2handle(self.handle), self.overlapped, transferred, wait)
+        res = _kernel32.GetOverlappedResult(_int2handle(self.handle), self.overlapped, transferred, wait != 0)
+        
         if res:
             err = ERROR_SUCCESS
         else:
@@ -124,14 +132,11 @@
         if err in (ERROR_SUCCESS, ERROR_MORE_DATA, ERROR_OPERATION_ABORTED):
             self.completed = 1
             self.pending = 0
-        elif res == ERROR_IO_INCOMPLETE:
-            pass
-        else:
+        elif res != ERROR_IO_INCOMPLETE:
             self.pending = 0
             raise _WinError(IOError)
 
         if self.completed and self.readbuffer:
-            assert _ffi.typeof(self.readbuffer) is _ffi.typeof("CHAR[]")
             if transferred[0] != len(self.readbuffer):
                 tempbuffer = _ffi.new("CHAR[]", transferred[0])
                 _ffi.memmove(tempbuffer, self.readbuffer, transferred[0])
@@ -142,7 +147,11 @@
         if not self.completed:
             raise ValueError("can't get read buffer before GetOverlappedResult() "
                         "signals the operation completed")
-        return _ffi.buffer(self.readbuffer)
+        if self.readbuffer:
+            result = _ffi.buffer(self.readbuffer)
+        else:
+            result = None
+        return result
 
     def cancel(self):
         ret = True
@@ -181,7 +190,8 @@
         err = 0
     else: 
         err = _kernel32.GetLastError()
-    if overlapped:
+
+    if use_overlapped:
         if not ret:
             if err == ERROR_IO_PENDING:
                 overlapped.pending = 1
@@ -189,7 +199,7 @@
                 return _WinError(IOError)
         return overlapped, err
 
-    if not ret and err != ERROR_MORE_DATA:
+    if (not ret) and err != ERROR_MORE_DATA:
         return _WinError(IOError)
     return buf, err
 
@@ -197,8 +207,7 @@
     written = _ffi.new("DWORD*")
     err = _ffi.new("DWORD*")
     use_overlapped = overlapped
-    overlapped = None
-
+    overlapped = None    
     if use_overlapped:
         overlapped = Overlapped(handle)
         if not overlapped:
@@ -207,7 +216,6 @@
         buf = overlapped.writebuffer
     else:
         buf = _ffi.new("CHAR[]", bytes(buffer))
-    
     if use_overlapped:
         ret = _kernel32.WriteFile(_int2handle(handle), buf , len(buf), written, overlapped.overlapped)
     else:
@@ -218,21 +226,17 @@
     else: 
         err = _kernel32.GetLastError()
 
-    if overlapped:
+    if use_overlapped:
         if not ret:
             if err == ERROR_IO_PENDING:
                 overlapped.pending = 1
-            elif err != ERROR_MORE_DATA:
+            else:
                 return _WinError(IOError)
         return overlapped, err
 
     if not ret:
         return _WinError(IOError)
      
-    # The whole of the buffer should have been written
-    # otherwise this function call has been be successful
-    assert written[0] == len(buf)
-
     return written[0], err
 
  
@@ -350,9 +354,7 @@
             # Not sure what that is doing currently.
             SetFromWindowsErr(0)
 
-        assert nread == len(buf)
- #       if (_PyBytes_Resize(&buf, nread))
- #           return NULL;
+
         return  buf, navail[0], nleft[0]
     else:
         ret = _kernel32.PeekNamedPipe(_int2handle(handle), _ffi.NULL, 0, _ffi.NULL, navail, nleft)
@@ -373,6 +375,7 @@
 def WaitNamedPipe(namedpipe, milliseconds):
     namedpipe = _ffi.new("CHAR[]", namedpipe.encode("ascii", "ignore"))
     res = _kernel32.WaitNamedPipeA(namedpipe, milliseconds)
+
     if res < 0:
         raise SetFromWindowsErr(0)
 
@@ -433,6 +436,9 @@
         raise _WinError()
     return _ffi.string(buf)
 
+def ExitProcess(exitcode):
+    _kernel32.ExitProcess(exitcode)
+
 ZERO_MEMORY = 0x00000008
 
 def malloc(size):

From pypy.commits at gmail.com  Sat May 11 15:34:15 2019
From: pypy.commits at gmail.com (andrewjlawrence)
Date: Sat, 11 May 2019 12:34:15 -0700 (PDT)
Subject: [pypy-commit] pypy winmultiprocessing: Merged in latest changes
 from py3.6
Message-ID: <5cd723b7.1c69fb81.2530d.ce3c@mx.google.com>

Author: andrewjlawrence
Branch: winmultiprocessing
Changeset: r96596:ff7e3469b931
Date: 2019-05-11 19:14 +0100
http://bitbucket.org/pypy/pypy/changeset/ff7e3469b931/

Log:	Merged in latest changes from py3.6

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -10,10 +10,6 @@
 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1
 10f1b29a2bd21f837090286174a9ca030b8680b2 release-2.5.0
 9c4588d731b7fe0b08669bd732c2b676cb0a8233 release-2.5.1
-fcdb941565156385cbac04cfb891f8f4c7a92ef6 release-2.6.0
-fcdb941565156385cbac04cfb891f8f4c7a92ef6 release-2.6.0
-e03971291f3a0729ecd3ee7fae7ddb0bb82d476c release-2.6.0
-e03971291f3a0729ecd3ee7fae7ddb0bb82d476c release-2.6.0
 295ee98b69288471b0fcf2e0ede82ce5209eb90b release-2.6.0
 f3ad1e1e1d6215e20d34bb65ab85ff9188c9f559 release-2.6.1
 850edf14b2c75573720f59e95767335fb1affe55 release-4.0.0
@@ -24,17 +20,10 @@
 b0a649e90b6642251fb4a765fe5b27a97b1319a9 release-5.1.1
 80ef432a32d9baa4b3c5a54c215e8ebe499f6374 release-5.1.2
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
-40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
 7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1
-68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
-68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
 77392ad263504df011ccfcabf6a62e21d04086d0 release-pypy2.7-v5.4.0
-050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
-050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
 0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1
-4909c06daf41ce88f87dc01c57959cadad4df4a8 RevDB-pypy2.7-v5.4.1
-4909c06daf41ce88f87dc01c57959cadad4df4a8 RevDB-pypy2.7-v5.4.1
 d7724c0a5700b895a47de44074cdf5fd659a988f RevDB-pypy2.7-v5.4.1
 aff251e543859ce4508159dd9f1a82a2f553de00 release-pypy2.7-v5.6.0
 e90317857d27917bf840caf675832292ee070510 RevDB-pypy2.7-v5.6.1
@@ -45,33 +34,20 @@
 2875f328eae2216a87f3d6f335092832eb031f56 release-pypy3.5-v5.7.1
 c925e73810367cd960a32592dd7f728f436c125c release-pypy2.7-v5.8.0
 a37ecfe5f142bc971a86d17305cc5d1d70abec64 release-pypy3.5-v5.8.0
-03d614975835870da65ff0481e1edad68ebbcb8d release-pypy2.7-v5.9.0
 d72f9800a42b46a8056951b1da2426d2c2d8d502 release-pypy3.5-v5.9.0
-03d614975835870da65ff0481e1edad68ebbcb8d release-pypy2.7-v5.9.0
 84a2f3e6a7f88f2fe698e473998755b3bd1a12e2 release-pypy2.7-v5.9.0
 0e7ea4fe15e82d5124e805e2e4a37cae1a402d4b release-pypy2.7-v5.10.0
-a91df6163fb76df245091f741dbf6a23ddc72374 release-pypy3.5-v5.10.0
-a91df6163fb76df245091f741dbf6a23ddc72374 release-pypy3.5-v5.10.0
-0000000000000000000000000000000000000000 release-pypy3.5-v5.10.0
-0000000000000000000000000000000000000000 release-pypy3.5-v5.10.0
 09f9160b643e3f02ccb8c843b2fbb4e5cbf54082 release-pypy3.5-v5.10.0
 3f6eaa010fce78cc7973bdc1dfdb95970f08fed2 release-pypy3.5-v5.10.1
 ab0b9caf307db6592905a80b8faffd69b39005b8 release-pypy2.7-v6.0.0
 fdd60ed87e941677e8ea11acf9f1819466521bf2 release-pypy3.5-v6.0.0
 9112c8071614108b1042bfef0713915107004d62 release-pypy2.7-v7.0.0
 1f86f25937b6ae6c8b25236c35228fac587678bf release-pypy3.5-v7.0.0
-dab365a465140aa79a5f3ba4db784c4af4d5c195 release-pypy3.6-v7.0.0
-9112c8071614108b1042bfef0713915107004d62 release-pypy2.7-v7.0.0
 c8805ee6d7846ca2722b106eeaa2f128c699aba3 release-pypy2.7-v7.0.0
-1f86f25937b6ae6c8b25236c35228fac587678bf release-pypy3.5-v7.0.0
 928a4f70d3de7d17449456946154c5da6e600162 release-pypy3.5-v7.0.0
-dab365a465140aa79a5f3ba4db784c4af4d5c195 release-pypy3.6-v7.0.0
 fb40f7a5524c77b80e6c468e087d621610137261 release-pypy3.6-v7.0.0
 990cef41fe11e5d46b019a46aa956ff46ea1a234 release-pypy2.7-v7.1.0
-bb0d05b190b9c579f0c889a368636e14f6205bab release-pypy3.6-v7.1.0
-bb0d05b190b9c579f0c889a368636e14f6205bab release-pypy3.6-v7.1.0
-6fd188f8f903b7555920adf7d5e7fe21db1bd593 release-pypy3.6-v7.1.0
-6fd188f8f903b7555920adf7d5e7fe21db1bd593 release-pypy3.6-v7.1.0
-7a2e437acfceafe2665b23b1394dc6c66add3b89 release-pypy3.6-v7.1.0
-7a2e437acfceafe2665b23b1394dc6c66add3b89 release-pypy3.6-v7.1.0
 de061d87e39c7df4e436974096d7982c676a859d release-pypy3.6-v7.1.0
+784b254d669919c872a505b807db8462b6140973 release-pypy3.6-v7.1.1
+8cdda8b8cdb8ff29d9e620cccd6c5edd2f2a23ec release-pypy2.7-v7.1.1
+
diff --git a/lib-python/3/pdb.py b/lib-python/3/pdb.py
--- a/lib-python/3/pdb.py
+++ b/lib-python/3/pdb.py
@@ -341,8 +341,14 @@
     def interaction(self, frame, traceback):
         # Restore the previous signal handler at the Pdb prompt.
         if Pdb._previous_sigint_handler:
-            signal.signal(signal.SIGINT, Pdb._previous_sigint_handler)
-            Pdb._previous_sigint_handler = None
+            try:
+                signal.signal(signal.SIGINT, Pdb._previous_sigint_handler)
+                Pdb._previous_sigint_handler = None
+            except ValueError:
+                # ValueError happens when we're in a non-main thread,
+                # if we already invoked pdb in the same program from the
+                # main thread.  (PyPy fix)
+                pass
         if self.setup(frame, traceback):
             # no interaction desired at this time (happens if .pdbrc contains
             # a command like "continue")
diff --git a/lib_pypy/_blake2/impl/blake2b.o b/lib_pypy/_blake2/impl/blake2b.o
deleted file mode 100644
index 0957bfc2e8b188b193b9eee3808641a638e0a2f7..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch

[cut]

diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -95,7 +95,9 @@
 There are a few extra implications from the difference in the GC.  Most
 notably, if an object has a ``__del__``, the ``__del__`` is never called more
 than once in PyPy; but CPython will call the same ``__del__`` several times
-if the object is resurrected and dies again.  The ``__del__`` methods are
+if the object is resurrected and dies again (at least it is reliably so in
+older CPythons; newer CPythons try to call destructors not more than once,
+but there are counter-examples).  The ``__del__`` methods are
 called in "the right" order if they are on objects pointing to each
 other, as in CPython, but unlike CPython, if there is a dead cycle of
 objects referencing each other, their ``__del__`` methods are called anyway;
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-v7.1.1.rst
    release-v7.1.0.rst
    release-v7.0.0.rst
    release-v6.0.0.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-7.1.0.rst
    whatsnew-pypy2-7.0.0.rst
    whatsnew-pypy2-6.0.0.rst
    whatsnew-pypy2-5.10.0.rst
diff --git a/pypy/doc/release-v7.1.1.rst b/pypy/doc/release-v7.1.1.rst
--- a/pypy/doc/release-v7.1.1.rst
+++ b/pypy/doc/release-v7.1.1.rst
@@ -68,10 +68,14 @@
 Changelog
 =========
 
-Changes shared across versions
-* improve performance of ``u''.append``
+Changes shared across versions:
+
+* Improve performance of ``u''.append``
+
 * Prevent a crash in ``zlib`` when flushing a closed stream
-* Fix a few corener cases when encountering unicode values above 0x110000
+
+* Fix a few corner cases when encountering unicode values above 0x110000
+
 * Teach the JIT how to handle very large constant lists, sets, or dicts
 * Fix building on ARM32 (issue 2984_)
 * Fix a bug in register assignment in ARM32
@@ -81,9 +85,9 @@
 * Fix memoryviews of ctype structures with padding, (cpython issue 32780_)
 * CFFI updated to as-yet-unreleased 1.12.3
 
-Python 3.6 only
+Python 3.6 only:
 
-* On win32, override some ``errno.E*`` values that were added to MSVC in v2010
+* Override some ``errno.E*`` values that were added to MSVC in v2010
   so that ``errno.E* == errno.WSAE*`` as in CPython
 * Do the same optimization that CPython does for ``(1, 2, 3, *a)`` (but at the
   AST level)
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -16,3 +16,12 @@
 .. branch: datetime_api_27
 
 Add ``DateTime_FromTimestamp`` and ``Date_FromTimestamp``
+
+.. branch: issue2968
+
+Fix segfault in cpyext_tp_new_tupl
+
+.. branch: semlock-deadlock
+
+Test and reduce the probability of a deadlock when acquiring a semaphore by
+moving global state changes closer to the actual aquire.
diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -180,14 +180,17 @@
         too_many_args = False
 
         # put the special w_firstarg into the scope, if it exists
+        upfront = 0
+        args_w = self.arguments_w
         if w_firstarg is not None:
-            upfront = 1
             if co_argcount > 0:
                 scope_w[0] = w_firstarg
-        else:
-            upfront = 0
+                upfront = 1
+            else:
+                # ugh, this is a call to a method 'def meth(*args)', maybe
+                # (see test_issue2996_*).  Fall-back solution...
+                args_w = [w_firstarg] + args_w
 
-        args_w = self.arguments_w
         num_args = len(args_w)
         avail = num_args + upfront
 
@@ -210,11 +213,8 @@
         # collect extra positional arguments into the *vararg
         if signature.has_vararg():
             args_left = co_argcount - upfront
-            if args_left < 0:  # check required by rpython
-                starargs_w = [w_firstarg]
-                if num_args:
-                    starargs_w = starargs_w + args_w
-            elif num_args > args_left:
+            assert args_left >= 0  # check required by rpython
+            if num_args > args_left:
                 starargs_w = args_w[args_left:]
             else:
                 starargs_w = []
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -509,8 +509,7 @@
             return
         self.update_position(asrt.lineno)
         end = self.new_block()
-        if self.compile_info.optimize != 0:
-            self.emit_jump(ops.JUMP_IF_NOT_DEBUG, end)
+        self.emit_jump(ops.JUMP_IF_NOT_DEBUG, end)
         asrt.test.accept_jump_if(self, True, end)
         self.emit_op_name(ops.LOAD_GLOBAL, self.names, "AssertionError")
         if asrt.msg:
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -58,8 +58,9 @@
     assert argcount >= 0     # annotator hint
     assert kwonlyargcount >= 0
     argnames = list(varnames[:argcount])
-    if argcount < len(varnames):
+    if kwonlyargcount > 0:
         kwonlyargs = list(varnames[argcount:argcount + kwonlyargcount])
+        argcount += kwonlyargcount
     else:
         kwonlyargs = None
     if code.co_flags & CO_VARARGS:
@@ -68,7 +69,7 @@
     else:
         varargname = None
     if code.co_flags & CO_VARKEYWORDS:
-        kwargname = code.co_varnames[argcount + kwonlyargcount]
+        kwargname = code.co_varnames[argcount]
     else:
         kwargname = None
     return Signature(argnames, varargname, kwargname, kwonlyargs)
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -923,3 +923,18 @@
             def test(**kwargs):
                 return kwargs
             assert test(**q) == {"foo": "bar"}
+
+    def test_issue2996_1(self): """
+        class Class:
+            def method(*args, a_parameter=None, **kwargs):
+                pass
+        Class().method(**{'a_parameter': 4})
+        """
+
+    def test_issue2996_2(self): """
+        class Foo:
+            def methhh(*args, offset=42):
+                return args, offset
+        foo = Foo()
+        assert foo.methhh(**{}) == ((foo,), 42)
+        """
diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py
--- a/pypy/interpreter/test/test_compiler.py
+++ b/pypy/interpreter/test/test_compiler.py
@@ -824,6 +824,13 @@
         sig = cpython_code_signature(co)
         assert sig == Signature(['a', 'b'], None, 'kwargs', ['m', 'n'])
 
+        # a variant with varargname, which was buggy before issue2996
+        snippet = 'def f(*args, offset=42): pass'
+        containing_co = self.compiler.compile(snippet, '<string>', 'single', 0)
+        co = find_func(containing_co)
+        sig = cpython_code_signature(co)
+        assert sig == Signature([], 'args', None, ['offset'])
+
 
 class AppTestCompiler(object):
 
diff --git a/pypy/interpreter/test/test_zpy.py b/pypy/interpreter/test/test_zpy.py
--- a/pypy/interpreter/test/test_zpy.py
+++ b/pypy/interpreter/test/test_zpy.py
@@ -84,7 +84,28 @@
     # test 3 : additionnal pypy parameters
     output = run(sys.executable, pypypath, '-S', "-O", tmpfilepath, "hello")
     assert output.splitlines()[-1] == str([tmpfilepath,'hello'])
-    
+
+def test_optimize_removes_assert():
+    tmpfilepath = str(udir.join("test_assert.py"))
+    tmpfile = file(tmpfilepath, "w")
+    tmpfile.write("""
+try:
+    assert 0
+except AssertionError:
+    print("AssertionError")
+else:
+    print("nothing")
+""")
+    tmpfile.close()
+
+    # no optimization: crashes
+    output = run(sys.executable, pypypath, '-S', tmpfilepath)
+    assert "AssertionError" in output
+
+    # optimization: just works
+    output = run(sys.executable, pypypath, '-SO', tmpfilepath)
+    assert "nothing" in output
+
 
 TB_NORMALIZATION_CHK= """
 class K(object):
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -1400,7 +1400,7 @@
                                   s, pos, pos + 4)
             result.append(r)
             continue
-        elif ch >= 0x110000:
+        elif r_uint(ch) >= 0x110000:
             r, pos, rettype = errorhandler(errors, public_encoding_name,
                                   "codepoint not in range(0x110000)",
                                   s, pos, len(s))
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -575,14 +575,7 @@
     if encoding is None:
         encoding = space.sys.defaultencoding
     w_encoder = space.getitem(lookup_codec(space, encoding), space.newint(0))
-    w_retval =  _call_codec(space, w_encoder, w_obj, "encoding", encoding, errors)
-    if not space.isinstance_w(w_retval, space.w_bytes):
-        raise oefmt(space.w_TypeError,
-                    "'%s' encoder returned '%T' instead of 'bytes'; "
-                    "use codecs.encode() to encode to arbitrary types",
-                    encoding,
-                    w_retval)
-    return w_retval
+    return _call_codec(space, w_encoder, w_obj, "encoding", encoding, errors)
 
 @unwrap_spec(errors='text_or_none')
 def readbuffer_encode(space, w_data, errors='strict'):
@@ -604,14 +597,7 @@
     if encoding is None:
         encoding = space.sys.defaultencoding
     w_decoder = space.getitem(lookup_codec(space, encoding), space.newint(1))
-    w_retval = _call_codec(space, w_decoder, w_obj, "decoding", encoding, errors)
-    if not isinstance(w_retval, W_UnicodeObject):
-        raise oefmt(space.w_TypeError,
-                    "'%s' decoder returned '%T' instead of 'str'; "
-                    "use codecs.decode() to decode to arbitrary types",
-                    encoding,
-                    w_retval)
-    return w_retval
+    return _call_codec(space, w_decoder, w_obj, "decoding", encoding, errors)
 
 @unwrap_spec(errors='text')
 def register_error(space, errors, w_handler):
@@ -842,7 +828,7 @@
             if not 0 <= x <= 0x10FFFF:
                 raise oefmt(space.w_TypeError,
                     "character mapping must be in range(0x110000)")
-            return rutf8.unichr_as_utf8(x)
+            return rutf8.unichr_as_utf8(x, allow_surrogates=True)
         elif space.is_w(w_ch, space.w_None):
             # Charmap may return None
             return errorchar
@@ -972,9 +958,17 @@
         unicode_name_handler)
 
     if first_escape_error_char is not None:
+        # Here, 'first_escape_error_char' is a single string character.
+        # Careful, it might be >= '\x80'.  If it is, it would made an
+        # invalid utf-8 string when pasted directory in it.
+        if ' ' <= first_escape_error_char < '\x7f':
+            msg = "invalid escape sequence '\\%s'" % (first_escape_error_char,)
+        else:
+            msg = "invalid escape sequence: '\\' followed by %s" % (
+                space.text_w(space.repr(
+                    space.newbytes(first_escape_error_char))),)
         space.warn(
-            space.newtext("invalid escape sequence '\\%s'"
-                          % str(first_escape_error_char)),
+            space.newtext(msg),
             space.w_DeprecationWarning
         )
     return space.newtuple([space.newutf8(result, lgt), space.newint(u_len)])
diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -125,6 +125,7 @@
         assert (charmap_decode(b"\x00\x01\x02", "strict",
                                {0: u'\U0010FFFF', 1: u'b', 2: u'c'}) ==
                 (u"\U0010FFFFbc", 3))
+        assert charmap_decode(b'\xff', "strict", {0xff: 0xd800}) == (u'\ud800', 1)
 
     def test_escape_decode(self):
         from _codecs import unicode_escape_decode as decode
@@ -1386,7 +1387,7 @@
                      "foo\udca5bar")
         assert ("foo\udca5bar".encode("iso-8859-3", "surrogateescape") == 
                          b"foo\xa5bar")
-    
+
     def test_warn_escape_decode(self):
         import warnings
         import codecs
@@ -1394,10 +1395,37 @@
         with warnings.catch_warnings(record=True) as l:
             warnings.simplefilter("always")
             codecs.unicode_escape_decode(b'\\A')
-            codecs.unicode_escape_decode(b"\\A")
+            codecs.unicode_escape_decode(b"\\" + b"\xff")
 
         assert len(l) == 2
         assert isinstance(l[0].message, DeprecationWarning)
+        assert isinstance(l[1].message, DeprecationWarning)
 
+    def test_invalid_type_errors(self):
+        # hex is not a text encoding. it works via the codecs functions, but
+        # not the methods
+        import codecs
+        res = codecs.decode(b"aabb", "hex")
+        assert res == b"\xaa\xbb"
+        res = codecs.decode(u"aabb", "hex")
+        assert res == b"\xaa\xbb"
+        res = codecs.encode(b"\xaa\xbb", "hex")
+        assert res == b"aabb"
 
+        raises(LookupError, u"abc".encode, "hex")
 
+    def test_non_text_codec(self):
+        import _codecs
+        def search_function(encoding):
+            def f(input, errors="strict"):
+                return 52, len(input)
+            if encoding == 'test.mynontextenc':
+                return (f, f, None, None)
+            return None
+        _codecs.register(search_function)
+        res = _codecs.encode(u"abc", "test.mynontextenc")
+        assert res == 52
+        res = _codecs.decode(b"abc", "test.mynontextenc")
+        assert res == 52
+        raises(TypeError, u"abc".encode, "test.mynontextenc")
+        raises(TypeError, b"abc".decode, "test.mynontextenc")
diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -105,16 +105,10 @@
         # desired, all in one pass.
         seennl = self.seennl
 
-        # If, up to now, newlines are consistently \n, do a quick check
-        # for the \r
-        only_lf = False
-        if seennl == SEEN_LF or seennl == 0:
-            only_lf = (output.find('\r') < 0)
-
-        if only_lf:
-            # If not already seen, quick scan for a possible "\n" character.
+        if output.find('\r') < 0:
+            # If no \r, quick scan for a possible "\n" character.
             # (there's nothing else to be done, even when in translation mode)
-            if seennl == 0 and output.find('\n') >= 0:
+            if output.find('\n') >= 0:
                 seennl |= SEEN_LF
                 # Finished: we have scanned for newlines, and none of them
                 # need translating.
diff --git a/pypy/module/_io/test/test_textio.py b/pypy/module/_io/test/test_textio.py
--- a/pypy/module/_io/test/test_textio.py
+++ b/pypy/module/_io/test/test_textio.py
@@ -566,3 +566,13 @@
         _check(dec)
         dec = _io.IncrementalNewlineDecoder(None, translate=True)
         _check(dec)
+
+    def test_newlines2(self):
+        import _io, codecs
+        inner_decoder = codecs.getincrementaldecoder("utf-8")()
+        decoder = _io.IncrementalNewlineDecoder(inner_decoder, translate=True)
+        msg = b"abc\r\n\n\r\r\n\n"
+        decoded = ''
+        for ch in msg:
+            decoded += decoder.decode(bytes([ch]))
+        assert set(decoder.newlines) == {"\r", "\n", "\r\n"}
diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -48,7 +48,8 @@
     eci = ExternalCompilationInfo(
         includes = ['sys/time.h',
                     'limits.h',
-                    'semaphore.h'],
+                    'semaphore.h',
+                    ],
         libraries = libraries,
         )
 
@@ -269,6 +270,8 @@
         res = rwin32.WaitForSingleObject(self.handle, 0)
 
         if res != rwin32.WAIT_TIMEOUT:
+            self.last_tid = rthread.get_ident()
+            self.count += 1
             return True
 
         msecs = full_msecs
@@ -301,6 +304,8 @@
 
         # handle result
         if res != rwin32.WAIT_TIMEOUT:
+            self.last_tid = rthread.get_ident()
+            self.count += 1
             return True
         return False
 
@@ -379,8 +384,9 @@
                     elif e.errno in (errno.EAGAIN, errno.ETIMEDOUT):
                         return False
                     raise
-                _check_signals(space)
-
+                _check_signals(space)    
+                self.last_tid = rthread.get_ident()
+                self.count += 1
                 return True
         finally:
             if deadline:
@@ -449,6 +455,7 @@
         self.count = 0
         self.maxvalue = maxvalue
         self.register_finalizer(space)
+        self.last_tid = -1
         self.name = name
 
     def name_get(self, space):
@@ -495,15 +502,15 @@
         if self.kind == RECURSIVE_MUTEX and self._ismine():
             self.count += 1
             return space.w_True
-
         try:
+            # sets self.last_tid and increments self.count
+            # those steps need to be as close as possible to
+            # acquiring the semlock for self._ismine() to support
+            # multiple threads 
             got = semlock_acquire(self, space, block, w_timeout)
         except OSError as e:
             raise wrap_oserror(space, e)
-
         if got:
-            self.last_tid = rthread.get_ident()
-            self.count += 1
             return space.w_True
         else:
             return space.w_False
@@ -520,10 +527,10 @@
 
         try:
             semlock_release(self, space)
+            self.count -= 1
         except OSError as e:
             raise wrap_oserror(space, e)
 
-        self.count -= 1
 
     def after_fork(self):
         self.count = 0
diff --git a/pypy/module/_multiprocessing/test/test_semaphore.py b/pypy/module/_multiprocessing/test/test_semaphore.py
--- a/pypy/module/_multiprocessing/test/test_semaphore.py
+++ b/pypy/module/_multiprocessing/test/test_semaphore.py
@@ -18,6 +18,7 @@
     def setup_class(cls):
         cls.w_SEMAPHORE = cls.space.wrap(SEMAPHORE)
         cls.w_RECURSIVE = cls.space.wrap(RECURSIVE_MUTEX)
+        cls.w_runappdirect = cls.space.wrap(cls.runappdirect)
 
     @py.test.mark.skipif("sys.platform == 'win32'")
     def test_sem_unlink(self):
@@ -138,3 +139,25 @@
         from _multiprocessing import SemLock
         sem = SemLock(self.SEMAPHORE, 1, 1, '/mp-123', unlink=True)
         assert sem._count() == 0
+
+    def test_in_threads(self):
+        from _multiprocessing import SemLock
+        from threading import Thread
+        from time import sleep
+        l = SemLock(0, 1, 1, "6", unlink=True)
+        if self.runappdirect:
+            def f(id):
+                for i in range(10000):
+                    pass
+        else:
+            def f(id):
+                for i in range(1000):
+                    # reduce the probability of thread switching
+                    # at exactly the wrong time in semlock_acquire
+                    for j in range(10):
+                        pass
+        threads = [Thread(None, f, args=(i,)) for i in range(2)]
+        [t.start() for t in threads]
+        # if the RLock calls to sem_wait and sem_post do not match,
+        # one of the threads will block and the call to join will fail
+        [t.join() for t in threads]
diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -366,10 +366,14 @@
         hexdigits = self.getslice(start, i)
         try:
             val = int(hexdigits, 16)
-            if sys.maxunicode > 65535 and 0xd800 <= val <= 0xdfff:
-                # surrogate pair
-                if self.ll_chars[i] == '\\' and self.ll_chars[i+1] == 'u':
-                    val = self.decode_surrogate_pair(i, val)
+            if (0xd800 <= val <= 0xdbff and
+                    self.ll_chars[i] == '\\' and self.ll_chars[i+1] == 'u'):
+                hexdigits = self.getslice(i+2, i+6)
+                lowsurr = int(hexdigits, 16)
+                if 0xdc00 <= lowsurr <= 0xdfff:
+                    # decode surrogate pair
+                    val = 0x10000 + (((val - 0xd800) << 10) |
+                                     (lowsurr - 0xdc00))
                     i += 6
         except ValueError:
             raise DecoderError("Invalid \uXXXX escape (char %d)", i-1)
@@ -380,15 +384,6 @@
         builder.append(utf8_ch)
         return i
 
-    def decode_surrogate_pair(self, i, highsurr):
-        """ uppon enter the following must hold:
-              chars[i] == "\\" and chars[i+1] == "u"
-        """
-        i += 2
-        hexdigits = self.getslice(i, i+4)
-        lowsurr = int(hexdigits, 16) # the possible ValueError is caugth by the caller
-        return 0x10000 + (((highsurr - 0xd800) << 10) | (lowsurr - 0xdc00))
-
     def decode_key(self, i):
         """ returns a wrapped unicode """
         from rpython.rlib.rarithmetic import intmask
diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py
--- a/pypy/module/_pypyjson/test/test__pypyjson.py
+++ b/pypy/module/_pypyjson/test/test__pypyjson.py
@@ -198,6 +198,17 @@
         res = _pypyjson.loads('"z\\ud834\\udd20x"')
         assert res == expected
 
+    def test_unicode_not_a_surrogate_pair(self):
+        import _pypyjson
+        res = _pypyjson.loads('"z\\ud800\\ud800x"')
+        assert list(res) == [u'z', u'\ud800', u'\ud800', u'x']
+        res = _pypyjson.loads('"z\\udbff\\uffffx"')
+        assert list(res) == [u'z', u'\udbff', u'\uffff', u'x']
+        res = _pypyjson.loads('"z\\ud800\\ud834\\udd20x"')
+        assert res == u'z\ud800\U0001d120x'
+        res = _pypyjson.loads('"z\\udc00\\udc00x"')
+        assert list(res) == [u'z', u'\udc00', u'\udc00', u'x']
+
     def test_lone_surrogate(self):
         import _pypyjson
         json = '{"a":"\\uD83D"}'
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -452,8 +452,13 @@
             elif c == 'c':
                 return space.newbytes(func(add_arg, argdesc, ll_type))
             elif c == 'u':
-                return space.newutf8(rutf8.unichr_as_utf8(
-                    r_uint(ord(func(add_arg, argdesc, ll_type)))), 1)
+                code = ord(func(add_arg, argdesc, ll_type))
+                try:
+                    return space.newutf8(rutf8.unichr_as_utf8(
+                        r_uint(code), allow_surrogates=True), 1)
+                except rutf8.OutOfRange:
+                    raise oefmt(space.w_ValueError,
+                        "unicode character %d out of range", code)
             elif c == 'f' or c == 'd' or c == 'g':
                 return space.newfloat(float(func(add_arg, argdesc, ll_type)))
             else:
diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py
--- a/pypy/module/_rawffi/test/test__rawffi.py
+++ b/pypy/module/_rawffi/test/test__rawffi.py
@@ -348,6 +348,21 @@
         arg2.free()
         a.free()
 
+    def test_unicode_array(self):
+        import _rawffi
+        A = _rawffi.Array('u')
+        a = A(6, u'\u1234')
+        assert a[0] == u'\u1234'
+        a[0] = u'\U00012345'
+        assert a[0] == u'\U00012345'
+        a[0] = u'\ud800'
+        assert a[0] == u'\ud800'
+        B = _rawffi.Array('i')
+        b = B.fromaddress(a.itemaddress(0), 1)
+        b[0] = 0xffffffff
+        raises(ValueError, "a[0]")
+        a.free()
+
     def test_returning_unicode(self):
         import _rawffi
         A = _rawffi.Array('u')
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -94,8 +94,8 @@
     lgt = min(arr1.len, arr2.len)
     for i in range(lgt):
         arr_eq_driver.jit_merge_point(comp_func=comp_op)
-        w_elem1 = arr1.w_getitem(space, i)
-        w_elem2 = arr2.w_getitem(space, i)
+        w_elem1 = arr1.w_getitem(space, i, integer_instead_of_char=True)
+        w_elem2 = arr2.w_getitem(space, i, integer_instead_of_char=True)
         if comp_op == EQ:
             res = space.eq_w(w_elem1, w_elem2)
             if not res:
@@ -1142,10 +1142,11 @@
             else:
                 self.fromsequence(w_iterable)
 
-        def w_getitem(self, space, idx):
+        def w_getitem(self, space, idx, integer_instead_of_char=False):
             item = self.get_buffer()[idx]
             keepalive_until_here(self)
-            if mytype.typecode in 'bBhHil':
+            if mytype.typecode in 'bBhHil' or (
+                    integer_instead_of_char and mytype.typecode in 'cu'):
                 item = rffi.cast(lltype.Signed, item)
                 return space.newint(item)
             if mytype.typecode in 'ILqQ':
@@ -1158,7 +1159,7 @@
             elif mytype.typecode == 'u':
                 code = r_uint(ord(item))
                 try:
-                    item = rutf8.unichr_as_utf8(code)
+                    item = rutf8.unichr_as_utf8(code, allow_surrogates=True)
                 except rutf8.OutOfRange:
                     raise oefmt(space.w_ValueError,
                         "cannot operate on this array('u') because it contains"
diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py
--- a/pypy/module/array/test/test_array.py
+++ b/pypy/module/array/test/test_array.py
@@ -892,14 +892,21 @@
         assert repr(mya('i', [1, 2, 3])) == "array('i', [1, 2, 3])"
         assert repr(mya('i', (1, 2, 3))) == "array('i', [1, 2, 3])"
 
+    def test_array_of_chars_equality(self):
+        input_bytes = '\x01\x63a\x00!'
+        a = self.array('c', input_bytes)
+        b = self.array('c', input_bytes)
+        b.byteswap()
+        assert a == b
+
     def test_unicode_outofrange(self):
         input_unicode = u'\x01\u263a\x00\ufeff'
         a = self.array('u', input_unicode)
         b = self.array('u', input_unicode)
         b.byteswap()
         assert b[2] == u'\u0000'
-        raises(ValueError, "b[1]")        # doesn't work
-        e = raises(ValueError, "a != b")  # doesn't work
+        assert a != b
+        e = raises(ValueError, "b[0]")        # doesn't work
         assert str(e.value) == (
             "cannot operate on this array('u') because it contains"
             " character U+1000000 not in range [U+0000; U+10ffff]"
@@ -910,6 +917,10 @@
         assert a.tounicode() == input_unicode
         raises(ValueError, b.tounicode)   # doesn't work
 
+    def test_unicode_surrogate(self):
+        a = self.array('u', u'\ud800')
+        assert a[0] == u'\ud800'
+
     def test_weakref(self):
         import weakref
         a = self.array('u', 'Hi!')
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -639,16 +639,18 @@
     'PyThread_ReInitTLS', 'PyThread_init_thread',
     'PyThread_start_new_thread',
 
-    'PyStructSequence_InitType', 'PyStructSequence_New',
-    'PyStructSequence_UnnamedField',
+    'PyStructSequence_InitType', 'PyStructSequence_InitType2',
+    'PyStructSequence_New', 'PyStructSequence_UnnamedField',
 
     'PyFunction_Type', 'PyMethod_Type', 'PyRange_Type', 'PyTraceBack_Type',
 
-    'Py_DebugFlag', 'Py_VerboseFlag', 'Py_InteractiveFlag', 'Py_InspectFlag',
+    'Py_DebugFlag', 'Py_VerboseFlag', 'Py_QuietFlag',
+    'Py_InteractiveFlag', 'Py_InspectFlag',
     'Py_OptimizeFlag', 'Py_NoSiteFlag', 'Py_BytesWarningFlag', 'Py_UseClassExceptionsFlag',
-    'Py_FrozenFlag', 'Py_TabcheckFlag', 'Py_UnicodeFlag', 'Py_IgnoreEnvironmentFlag',
-    'Py_DivisionWarningFlag', 'Py_DontWriteBytecodeFlag', 'Py_NoUserSiteDirectory',
-    '_Py_QnewFlag', 'Py_Py3kWarningFlag', 'Py_HashRandomizationFlag', '_Py_PackageContext',
+    'Py_FrozenFlag', 'Py_IgnoreEnvironmentFlag',
+    'Py_DontWriteBytecodeFlag', 'Py_NoUserSiteDirectory',
+    'Py_UnbufferedStdioFlag', 'Py_HashRandomizationFlag', 'Py_IsolatedFlag',
+    '_Py_PackageContext',
     'PyOS_InputHook',
 
     'PyMem_RawMalloc', 'PyMem_RawCalloc', 'PyMem_RawRealloc', 'PyMem_RawFree',
@@ -664,6 +666,8 @@
     'PyObject_Init', 'PyObject_InitVar',
     'PyTuple_New', '_Py_Dealloc',
 ]
+if sys.platform == "win32":
+    SYMBOLS_C.append('Py_LegacyWindowsStdioFlag')
 TYPES = {}
 FORWARD_DECLS = []
 INIT_FUNCTIONS = []
diff --git a/pypy/module/cpyext/include/pythonrun.h b/pypy/module/cpyext/include/pythonrun.h
--- a/pypy/module/cpyext/include/pythonrun.h
+++ b/pypy/module/cpyext/include/pythonrun.h
@@ -9,6 +9,7 @@
 PyAPI_FUNC(void) Py_FatalError(const char *msg);
 
 /* taken from Python-3.2.3/Include/pydebug.h */
+/* Note: they are always 0 for now, expect Py_DebugFlag which is always 1 */
 PyAPI_DATA(int) Py_DebugFlag;
 PyAPI_DATA(int) Py_VerboseFlag;
 PyAPI_DATA(int) Py_QuietFlag;
@@ -20,11 +21,17 @@
 PyAPI_DATA(int) Py_UseClassExceptionsFlag;
 PyAPI_DATA(int) Py_FrozenFlag;
 PyAPI_DATA(int) Py_IgnoreEnvironmentFlag;
-PyAPI_DATA(int) Py_DivisionWarningFlag;
 PyAPI_DATA(int) Py_DontWriteBytecodeFlag;
 PyAPI_DATA(int) Py_NoUserSiteDirectory;
 PyAPI_DATA(int) Py_UnbufferedStdioFlag;
 PyAPI_DATA(int) Py_HashRandomizationFlag;
+PyAPI_DATA(int) Py_IsolatedFlag;
+
+#ifdef MS_WINDOWS
+PyAPI_DATA(int) Py_LegacyWindowsStdioFlag;
+#endif
+
+#define Py_GETENV(s) (Py_IgnoreEnvironmentFlag ? NULL : getenv(s))
 
 
 typedef struct {
diff --git a/pypy/module/cpyext/include/structseq.h b/pypy/module/cpyext/include/structseq.h
--- a/pypy/module/cpyext/include/structseq.h
+++ b/pypy/module/cpyext/include/structseq.h
@@ -24,6 +24,9 @@
 PyAPI_FUNC(void) PyStructSequence_InitType(PyTypeObject *type,
                                            PyStructSequence_Desc *desc);
 
+PyAPI_FUNC(int) PyStructSequence_InitType2(PyTypeObject *type,
+                                           PyStructSequence_Desc *desc);
+
 PyAPI_FUNC(PyObject *) PyStructSequence_New(PyTypeObject* type);
 
 typedef struct {
diff --git a/pypy/module/cpyext/src/missing.c b/pypy/module/cpyext/src/missing.c
--- a/pypy/module/cpyext/src/missing.c
+++ b/pypy/module/cpyext/src/missing.c
@@ -10,6 +10,7 @@
 
 int Py_DebugFlag = 1;
 int Py_VerboseFlag = 0;
+int Py_QuietFlag = 0;
 int Py_InteractiveFlag = 0;
 int Py_InspectFlag = 0;
 int Py_OptimizeFlag = 0;
@@ -17,15 +18,17 @@
 int Py_BytesWarningFlag = 0;
 int Py_UseClassExceptionsFlag = 0;
 int Py_FrozenFlag = 0;
-int Py_TabcheckFlag = 0;
-int Py_UnicodeFlag = 0;
 int Py_IgnoreEnvironmentFlag = 0;
-int Py_DivisionWarningFlag = 0;
 int Py_DontWriteBytecodeFlag = 0;
 int Py_NoUserSiteDirectory = 0;
-int _Py_QnewFlag = 0;
-int Py_Py3kWarningFlag = 0;
+int Py_UnbufferedStdioFlag = 0;
 int Py_HashRandomizationFlag = 0;
+int Py_IsolatedFlag = 0;
+
+#ifdef MS_WINDOWS
+int Py_LegacyWindowsStdioFlag = 0;
+#endif
+
 
 const char *Py_FileSystemDefaultEncoding;  /* filled when cpyext is imported */
 void _Py_setfilesystemdefaultencoding(const char *enc) {
diff --git a/pypy/module/cpyext/src/structseq.c b/pypy/module/cpyext/src/structseq.c
--- a/pypy/module/cpyext/src/structseq.c
+++ b/pypy/module/cpyext/src/structseq.c
@@ -315,8 +315,8 @@
     structseq_new,                              /* tp_new */
 };
 
-void
-PyStructSequence_InitType(PyTypeObject *type, PyStructSequence_Desc *desc)
+int
+PyStructSequence_InitType2(PyTypeObject *type, PyStructSequence_Desc *desc)
 {
     PyObject *dict;
     PyMemberDef* members;
@@ -342,8 +342,10 @@
     type->tp_doc = desc->doc;
 
     members = PyMem_NEW(PyMemberDef, n_members-n_unnamed_members+1);
-    if (members == NULL)
-        return;
+    if (members == NULL) {
+        PyErr_NoMemory();
+        return -1;
+    }
 
     for (i = k = 0; i < n_members; ++i) {
         if (desc->fields[i].name == PyStructSequence_UnnamedField)
@@ -361,22 +363,33 @@
     type->tp_members = members;
 
     if (PyType_Ready(type) < 0)
-        return;
+        return -1;
     Py_INCREF(type);
 
     dict = type->tp_dict;
-#define SET_DICT_FROM_INT(key, value)                           \
+#define SET_DICT_FROM_SIZE(key, value)                          \
     do {                                                        \
-        PyObject *v = PyLong_FromLong((long) value);            \
-        if (v != NULL) {                                        \
-            PyDict_SetItemString(dict, key, v);                 \
+        PyObject *v = PyLong_FromSsize_t(value);                \
+        if (v == NULL)                                          \
+            return -1;                                          \
+        if (PyDict_SetItemString(dict, key, v) < 0) {           \
             Py_DECREF(v);                                       \
+            return -1;                                          \
         }                                                       \
+        Py_DECREF(v);                                           \
     } while (0)
 
-    SET_DICT_FROM_INT(visible_length_key, desc->n_in_sequence);
-    SET_DICT_FROM_INT(real_length_key, n_members);
-    SET_DICT_FROM_INT(unnamed_fields_key, n_unnamed_members);
+    SET_DICT_FROM_SIZE(visible_length_key, desc->n_in_sequence);
+    SET_DICT_FROM_SIZE(real_length_key, n_members);
+    SET_DICT_FROM_SIZE(unnamed_fields_key, n_unnamed_members);
+
+    return 0;
+}
+
+void
+PyStructSequence_InitType(PyTypeObject *type, PyStructSequence_Desc *desc)
+{
+    (void)PyStructSequence_InitType2(type, desc);
 }
 
 PyTypeObject*
diff --git a/pypy/module/zlib/interp_zlib.py b/pypy/module/zlib/interp_zlib.py
--- a/pypy/module/zlib/interp_zlib.py
+++ b/pypy/module/zlib/interp_zlib.py
@@ -344,8 +344,7 @@
                 raise oefmt(space.w_ValueError,
                             "length must be greater than zero")
         if not self.stream:
-            raise zlib_error(space,
-                             "compressor object already flushed")
+            return space.newbytes('')
         data = self.unconsumed_tail
         try:
             self.lock()
diff --git a/pypy/module/zlib/test/test_zlib.py b/pypy/module/zlib/test/test_zlib.py
--- a/pypy/module/zlib/test/test_zlib.py
+++ b/pypy/module/zlib/test/test_zlib.py
@@ -421,4 +421,5 @@
         dco = zlib.decompressobj()
         dco.decompress(x)
         dco.flush()
-        raises(self.zlib.error, dco.flush)
+        # multiple flush calls should not raise
+        dco.flush()
diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -326,7 +326,8 @@
             space = self.space
             if do_unicode:
                 cp = rutf8.codepoint_at_pos(self.fmt, self.fmtpos - 1)
-                w_s = space.newutf8(rutf8.unichr_as_utf8(r_uint(cp)), 1)
+                w_s = space.newutf8(rutf8.unichr_as_utf8(r_uint(cp),
+                                                  allow_surrogates=True), 1)
             else:
                 cp = ord(self.fmt[self.fmtpos - 1])
                 w_s = space.newbytes(chr(cp))
@@ -478,7 +479,8 @@
                 n = space.int_w(w_value)
                 if do_unicode:
                     try:
-                        c = rutf8.unichr_as_utf8(r_uint(n))
+                        c = rutf8.unichr_as_utf8(r_uint(n),
+                                                 allow_surrogates=True)
                     except rutf8.OutOfRange:
                         raise oefmt(space.w_OverflowError,
                                     "unicode character code out of range")
diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py
--- a/pypy/objspace/std/newformat.py
+++ b/pypy/objspace/std/newformat.py
@@ -357,9 +357,11 @@
             if recursive:
                 spec = self._build_string(spec_start, end, level)
             w_rendered = self.space.format(w_obj, self.wrap(spec))
-            unwrapper = "utf8_w" if self.is_unicode else "bytes_w"
-            to_interp = getattr(self.space, unwrapper)
-            return to_interp(w_rendered)
+            if self.is_unicode:
+                w_rendered = self.space.unicode_from_object(w_rendered)
+                return self.space.utf8_w(w_rendered)
+            else:
+                return self.space.bytes_w(w_rendered)
 
         def formatter_parser(self):
             self.parser_list_w = []
diff --git a/pypy/objspace/std/test/test_newformat.py b/pypy/objspace/std/test/test_newformat.py
--- a/pypy/objspace/std/test/test_newformat.py
+++ b/pypy/objspace/std/test/test_newformat.py
@@ -245,6 +245,7 @@
     def test_simple(self):
         assert format(self.i(2)) == "2"
         assert isinstance(format(self.i(2), ""), str)
+        assert isinstance(self.i(2).__format__(""), str)
 
     def test_invalid(self):
         raises(ValueError, format, self.i(8), "s")
@@ -491,3 +492,9 @@
         excinfo = raises(ValueError, "{:j}".format, x(1))
         print(excinfo.value)
         assert str(excinfo.value) == "Unknown format code j for object of type 'x'"
+
+    def test_format_char(self):
+        import sys
+        assert '{0:c}'.format(42) == '*'
+        assert '{0:c}'.format(1234) == '\u04d2'
+        raises(OverflowError, '{0:c}'.format, -1)
diff --git a/pypy/objspace/std/test/test_stringformat.py b/pypy/objspace/std/test/test_stringformat.py
--- a/pypy/objspace/std/test/test_stringformat.py
+++ b/pypy/objspace/std/test/test_stringformat.py
@@ -215,6 +215,7 @@
 
     def test_format_wrong_char(self):
         raises(ValueError, 'a%Zb'.__mod__, ((23,),))
+        raises(ValueError, u'a%\ud800b'.__mod__, ((23,),))
 
     def test_incomplete_format(self):
         raises(ValueError, '%'.__mod__, ((23,),))
@@ -234,6 +235,8 @@
         raises(TypeError, '%c'.__mod__, ("",))
         raises(TypeError, '%c'.__mod__, (['c'],))
         raises(TypeError, '%c'.__mod__, b'A')
+        surrogate = 0xd800
+        assert '%c' % surrogate == '\ud800'
 
     def test___int__index__(self):
         class MyInt(object):
diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -728,6 +728,7 @@
 
         raises(TypeError, 'hello'.translate)
         raises(ValueError, "\xff".translate, {0xff: sys.maxunicode+1})
+        raises(TypeError, u'x'.translate, {ord('x'):0x110000})
 
     def test_maketrans(self):
         assert 'abababc' == 'abababc'.translate({'b': '<i>'})
@@ -1179,8 +1180,7 @@
     def test_format_repeat(self):
         assert format(u"abc", u"z<5") == u"abczz"
         assert format(u"abc", u"\u2007<5") == u"abc\u2007\u2007"
-        #CPython2 raises UnicodeEncodeError
-        assert format(123, u"\u2007<5") == u"123\u2007\u2007"
+        assert format(123, "\u2007<5") == "123\u2007\u2007"
 
     def test_formatting_unicode__repr__(self):
         # Printable character
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -1235,7 +1235,14 @@
                     a.pos, a.pos + 1)
                 assert False, "always raises"
             return space.newbytes(utf8)
-    return encode(space, w_obj, encoding, errors)
+    w_retval = encode(space, w_obj, encoding, errors)
+    if not space.isinstance_w(w_retval, space.w_bytes):
+        raise oefmt(space.w_TypeError,
+                    "'%s' encoder returned '%T' instead of 'bytes'; "
+                    "use codecs.encode() to encode to arbitrary types",
+                    encoding,
+                    w_retval)
+    return w_retval
 
 
 def decode_object(space, w_obj, encoding, errors=None):
@@ -1250,7 +1257,14 @@
             lgt = unicodehelper.check_utf8_or_raise(space, s)
             return space.newutf8(s, lgt)
     from pypy.module._codecs.interp_codecs import decode
-    return decode(space, w_obj, encoding, errors)
+    w_retval = decode(space, w_obj, encoding, errors)
+    if not isinstance(w_retval, W_UnicodeObject):
+        raise oefmt(space.w_TypeError,
+                    "'%s' decoder returned '%T' instead of 'str'; "
+                    "use codecs.decode() to decode to arbitrary types",
+                    encoding,
+                    w_retval)
+    return w_retval
 
 def unicode_from_object(space, w_obj):
     if space.is_w(space.type(w_obj), space.w_unicode):
diff --git a/pypy/tool/build_cffi_imports.py b/pypy/tool/build_cffi_imports.py
--- a/pypy/tool/build_cffi_imports.py
+++ b/pypy/tool/build_cffi_imports.py
@@ -7,6 +7,8 @@
 
 
 cffi_build_scripts = {
+    "_blake2": "_blake2/_blake2_build.py",
+    "_ssl": "_ssl_build.py",
     "sqlite3": "_sqlite3_build.py",
     "audioop": "_audioop_build.py",
     "tk": "_tkinter/tklib_build.py",
@@ -17,8 +19,6 @@
     "resource": "_resource_build.py" if sys.platform != "win32" else None,
     "lzma": "_lzma_build.py",
     "_decimal": "_decimal_build.py",
-    "_ssl": "_ssl_build.py",
-    "_blake2": "_blake2/_blake2_build.py",
     "_sha3": "_sha3/_sha3_build.py",
     "xx": None,    # for testing: 'None' should be completely ignored
     }
@@ -154,7 +154,7 @@
         status, stdout, stderr = run_subprocess(str(pypy_c), ['-m', 'ensurepip'])
     failures = []
 
-    for key, module in sorted(cffi_build_scripts.items()):
+    for key, module in cffi_build_scripts.items():
         if only and key not in only:
             print("* SKIPPING", key, '(not specified in --only)')
             continue
diff --git a/pypy/tool/release/force-builds.py b/pypy/tool/release/force-builds.py
--- a/pypy/tool/release/force-builds.py
+++ b/pypy/tool/release/force-builds.py
@@ -8,8 +8,13 @@
 
 modified by PyPy team
 """
+from __future__ import absolute_import, division, print_function
 
-import os, sys, urllib, subprocess
+import os, sys, subprocess
+try:
+    from urllib2 import quote
+except ImportError:
+    from urllib.request import quote
 
 from twisted.internet import reactor, defer
 from twisted.python import log
@@ -29,10 +34,10 @@
     'pypy-c-jit-macosx-x86-64',
     'pypy-c-jit-win-x86-32',
     'pypy-c-jit-linux-s390x',
-    'build-pypy-c-jit-linux-armhf-raspbian',
-    'build-pypy-c-jit-linux-armel',
+#    'build-pypy-c-jit-linux-armhf-raspbian',
+#    'build-pypy-c-jit-linux-armel',
     'rpython-linux-x86-32',
-    'rpython-linux-x86-64'
+    'rpython-linux-x86-64',
     'rpython-win-x86-32'
 ]
 
@@ -54,7 +59,7 @@
         log.err(err, "Build force failure")
 
     for builder in BUILDERS:
-        print 'Forcing', builder, '...'
+        print('Forcing', builder, '...')
         url = "http://" + server + "/builders/" + builder + "/force"
         args = [
             ('username', user),
@@ -63,15 +68,15 @@
             ('submit', 'Force Build'),
             ('branch', branch),
             ('comments', "Forced by command line script")]
-        url = url + '?' + '&'.join([k + '=' + urllib.quote(v) for (k, v) in args])
+        url = url + '?' + '&'.join([k + '=' + quote(v) for (k, v) in args])
         requests.append(
-            lock.run(client.getPage, url, followRedirect=False).addErrback(ebList))
+            lock.run(client.getPage, url.encode('utf-8'), followRedirect=False).addErrback(ebList))
 
     d = defer.gatherResults(requests)
     d.addErrback(log.err)
     d.addCallback(lambda ign: reactor.stop())
     reactor.run()
-    print 'See http://buildbot.pypy.org/summary after a while'
+    print('See http://buildbot.pypy.org/summary after a while')
 
 if __name__ == '__main__':
     log.startLogging(sys.stdout)
@@ -86,6 +91,6 @@
     try:
         subprocess.check_call(['hg','id','-r', options.branch])
     except subprocess.CalledProcessError:
-        print 'branch',  options.branch, 'could not be found in local repository'
+        print('branch',  options.branch, 'could not be found in local repository')
         sys.exit(-1) 
     main(options.branch, options.server, user=options.user)

From pypy.commits at gmail.com  Sat May 11 19:11:39 2019
From: pypy.commits at gmail.com (mattip)
Date: Sat, 11 May 2019 16:11:39 -0700 (PDT)
Subject: [pypy-commit] pypy cffi-libs: move _ssl and others to lib_pypy as
 cffi modules
Message-ID: <5cd756ab.1c69fb81.6f638.4718@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cffi-libs
Changeset: r96597:0ca2a46a6689
Date: 2019-05-09 20:04 -0700
http://bitbucket.org/pypy/pypy/changeset/0ca2a46a6689/

Log:	move _ssl and others to lib_pypy as cffi modules


From pypy.commits at gmail.com  Sat May 11 19:11:41 2019
From: pypy.commits at gmail.com (mattip)
Date: Sat, 11 May 2019 16:11:41 -0700 (PDT)
Subject: [pypy-commit] pypy cffi-libs: remove _ssl and dead code from module
 options
Message-ID: <5cd756ad.1c69fb81.207a1.7480@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cffi-libs
Changeset: r96598:d03b3d0c6b2e
Date: 2019-05-10 06:08 -0700
http://bitbucket.org/pypy/pypy/changeset/d03b3d0c6b2e/

Log:	remove _ssl and dead code from module options

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -33,10 +33,11 @@
     "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd",
     "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios",
     "zlib", "bz2", "struct", "_hashlib", "_md5", "_sha", "_minimal_curses",
-    "cStringIO", "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array",
+    "cStringIO", "thread", "itertools", "pyexpat", "cpyext", "array",
     "binascii", "_multiprocessing", '_warnings', "_collections",
     "_multibytecodec", "micronumpy", "_continuation", "_cffi_backend",
-    "_csv", "_cppyy", "_pypyjson", "_jitlog"
+    "_csv", "_cppyy", "_pypyjson", "_jitlog",
+    # _ssl
 ])
 
 import rpython.rlib.rvmprof.cintf
@@ -109,43 +110,6 @@
     module_suggests["cpyext"].append(("translation.shared", True))
 
 
-# NOTE: this dictionary is not used any more
-module_import_dependencies = {
-    # no _rawffi if importing rpython.rlib.clibffi raises ImportError
-    # or CompilationError or py.test.skip.Exception
-    "_rawffi"   : ["rpython.rlib.clibffi"],
-
-    "zlib"      : ["rpython.rlib.rzlib"],
-    "bz2"       : ["pypy.module.bz2.interp_bz2"],
-    "pyexpat"   : ["pypy.module.pyexpat.interp_pyexpat"],
-    "_ssl"      : ["pypy.module._ssl.interp_ssl"],
-    "_hashlib"  : ["pypy.module._ssl.interp_ssl"],
-    "_minimal_curses": ["pypy.module._minimal_curses.fficurses"],
-    "_continuation": ["rpython.rlib.rstacklet"],
-    "_vmprof"      : ["pypy.module._vmprof.interp_vmprof"],
-    "faulthandler" : ["pypy.module._vmprof.interp_vmprof"],
-    }
-
-def get_module_validator(modname):
-    # NOTE: this function is not used any more
-    if modname in module_import_dependencies:
-        modlist = module_import_dependencies[modname]
-        def validator(config):
-            from rpython.rtyper.tool.rffi_platform import CompilationError
-            try:
-                for name in modlist:
-                    __import__(name)
-            except (ImportError, CompilationError, py.test.skip.Exception) as e:
-                errcls = e.__class__.__name__
-                raise Exception(
-                    "The module %r is disabled\n" % (modname,) +
-                    "because importing %s raised %s\n" % (name, errcls) +
-                    str(e))
-        return validator
-    else:
-        return None
-
-
 pypy_optiondescription = OptionDescription("objspace", "Object Space Options", [
     OptionDescription("usemodules", "Which Modules should be used", [
         BoolOption(modname, "use module %s" % (modname, ),
@@ -154,7 +118,7 @@
                    requires=module_dependencies.get(modname, []),
                    suggests=module_suggests.get(modname, []),
                    negation=modname not in essential_modules,
-                   ) #validator=get_module_validator(modname))
+                   )
         for modname in all_modules]),
 
     BoolOption("allworkingmodules", "use as many working modules as possible",

From pypy.commits at gmail.com  Sat May 11 19:11:43 2019
From: pypy.commits at gmail.com (mattip)
Date: Sat, 11 May 2019 16:11:43 -0700 (PDT)
Subject: [pypy-commit] pypy cffi-libs: add _ssl and _cffi_ssl from py3.6
Message-ID: <5cd756af.1c69fb81.c597.9e70@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cffi-libs
Changeset: r96599:b7f0f92d6f30
Date: 2019-05-10 06:09 -0700
http://bitbucket.org/pypy/pypy/changeset/b7f0f92d6f30/

Log:	add _ssl and _cffi_ssl from py3.6

diff too long, truncating to 2000 out of 3406 lines

diff --git a/lib_pypy/_cffi_ssl/LICENSE b/lib_pypy/_cffi_ssl/LICENSE
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/LICENSE
@@ -0,0 +1,26 @@
+
+Except when otherwise stated (look for LICENSE files in directories or
+information at the beginning of each file) all software and
+documentation is licensed as follows: 
+
+    The MIT License
+
+    Permission is hereby granted, free of charge, to any person 
+    obtaining a copy of this software and associated documentation 
+    files (the "Software"), to deal in the Software without 
+    restriction, including without limitation the rights to use, 
+    copy, modify, merge, publish, distribute, sublicense, and/or 
+    sell copies of the Software, and to permit persons to whom the 
+    Software is furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included 
+    in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+    OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
+    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+    DEALINGS IN THE SOFTWARE.
+
diff --git a/lib_pypy/_cffi_ssl/README.md b/lib_pypy/_cffi_ssl/README.md
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/README.md
@@ -0,0 +1,35 @@
+# PyPy's SSL module
+
+All of the CFFI code is copied from cryptography, wich patches contributed
+back to cryptography. PyPy vendors it's own copy of the cffi backend thus
+it renames the compiled shared object to _pypy_openssl.so (which means
+that cryptography can ship their own cffi backend)
+
+NOTE: currently, we have the following changes:
+
+* ``_cffi_src/openssl/callbacks.py`` to not rely on the CPython C API
+  (this change is now backported)
+
+* ``_cffi_src/utils.py`` for issue #2575 (29c9a89359e4)
+
+* ``_cffi_src/openssl/x509_vfy.py`` for issue #2605 (ca4d0c90f5a1)
+
+* ``_cffi_src/openssl/pypy_win32_extra.py`` for Win32-only functionality like ssl.enum_certificates()
+
+
+# Tests?
+
+Currently this module is tested using CPython's standard library test suite.
+
+# Install it into PyPy's source tree
+
+Copy over all the sources into the folder `lib_pypy/_cffi_ssl/*`. Updating the cffi backend can be simply done by the following command::
+
+    $ cp -r <cloned cryptography folder>/src/_cffi_src/* .
+
+NOTE: you need to keep our version of ``_cffi_src/openssl/callbacks.py``
+for now!
+
+# Crpytography version
+
+Copied over release version `1.7.2`
diff --git a/lib_pypy/_cffi_ssl/__init__.py b/lib_pypy/_cffi_ssl/__init__.py
new file mode 100644
diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -0,0 +1,1566 @@
+import sys
+import time
+import _thread
+import socket
+import weakref
+from _pypy_openssl import ffi
+from _pypy_openssl import lib
+from _cffi_ssl._stdssl.certificate import (_test_decode_cert,
+    _decode_certificate, _certificate_to_der)
+from _cffi_ssl._stdssl.utility import (_str_with_len, _bytes_with_len,
+    _str_to_ffi_buffer, _str_from_buf, _cstr_decode_fs)
+from _cffi_ssl._stdssl.error import (ssl_error, pyssl_error,
+        SSLError, SSLZeroReturnError, SSLWantReadError,
+        SSLWantWriteError, SSLSyscallError,
+        SSLEOFError)
+from _cffi_ssl._stdssl.error import (SSL_ERROR_NONE,
+        SSL_ERROR_SSL, SSL_ERROR_WANT_READ, SSL_ERROR_WANT_WRITE,
+        SSL_ERROR_WANT_X509_LOOKUP, SSL_ERROR_SYSCALL,
+        SSL_ERROR_ZERO_RETURN, SSL_ERROR_WANT_CONNECT,
+        SSL_ERROR_EOF, SSL_ERROR_NO_SOCKET, SSL_ERROR_INVALID_ERROR_CODE,
+        pyerr_write_unraisable)
+from _cffi_ssl._stdssl import error
+from select import select
+from enum import IntEnum as _IntEnum
+
+if sys.platform == 'win32':
+    from _cffi_ssl._stdssl.win32_extra import enum_certificates, enum_crls
+    HAVE_POLL = False
+else:
+    from select import poll, POLLIN, POLLOUT
+    HAVE_POLL = True
+
+OPENSSL_VERSION = ffi.string(lib.OPENSSL_VERSION_TEXT).decode('utf-8')
+OPENSSL_VERSION_NUMBER = lib.OPENSSL_VERSION_NUMBER
+ver = OPENSSL_VERSION_NUMBER
+ver, status = divmod(ver, 16)
+ver, patch  = divmod(ver, 256)
+ver, fix    = divmod(ver, 256)
+ver, minor  = divmod(ver, 256)
+ver, major  = divmod(ver, 256)
+version_info = (major, minor, fix, patch, status)
+OPENSSL_VERSION_INFO = version_info
+_OPENSSL_API_VERSION = version_info
+del ver, version_info, status, patch, fix, minor, major
+
+HAS_ECDH = bool(lib.Cryptography_HAS_ECDH)
+HAS_SNI = bool(lib.Cryptography_HAS_TLSEXT_HOSTNAME)
+HAS_ALPN = bool(lib.Cryptography_HAS_ALPN)
+HAS_NPN = bool(lib.OPENSSL_NPN_NEGOTIATED)
+HAS_TLS_UNIQUE = True
+
+CLIENT = 0
+SERVER = 1
+
+VERIFY_DEFAULT = 0
+VERIFY_CRL_CHECK_LEAF = lib.X509_V_FLAG_CRL_CHECK
+VERIFY_CRL_CHECK_CHAIN = lib.X509_V_FLAG_CRL_CHECK | lib.X509_V_FLAG_CRL_CHECK_ALL
+VERIFY_X509_STRICT = lib.X509_V_FLAG_X509_STRICT
+if lib.Cryptography_HAS_X509_V_FLAG_TRUSTED_FIRST:
+    VERIFY_X509_TRUSTED_FIRST = lib.X509_V_FLAG_TRUSTED_FIRST
+
+CERT_NONE = 0
+CERT_OPTIONAL = 1
+CERT_REQUIRED = 2
+
+for name in dir(lib):
+    if name.startswith('SSL_OP'):
+        value = getattr(lib, name)
+        if value != 0:
+            globals()[name[4:]] = getattr(lib, name)
+
+OP_ALL = lib.SSL_OP_ALL & ~lib.SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS
+
+SSL_CLIENT = 0
+SSL_SERVER = 1
+
+SSL_CB_MAXLEN=128
+
+if lib.Cryptography_HAS_SSL2:
+    PROTOCOL_SSLv2  = 0
+PROTOCOL_SSLv3  = 1
+PROTOCOL_SSLv23 = 2
+PROTOCOL_TLS    = PROTOCOL_SSLv23
+PROTOCOL_TLSv1    = 3
+if lib.Cryptography_HAS_TLSv1_2:
+    PROTOCOL_TLSv1 = 3
+    PROTOCOL_TLSv1_1 = 4
+    PROTOCOL_TLSv1_2 = 5
+PROTOCOL_TLS_CLIENT = 0x10
+PROTOCOL_TLS_SERVER = 0x11
+
+_PROTOCOL_NAMES = (name for name in dir(lib) if name.startswith('PROTOCOL_'))
+
+_IntEnum._convert('_SSLMethod', __name__,
+        lambda name: name.startswith('PROTOCOL_'))
+
+if HAS_TLS_UNIQUE:
+    CHANNEL_BINDING_TYPES = ['tls-unique']
+else:
+    CHANNEL_BINDING_TYPES = []
+
+for name in error.SSL_AD_NAMES:
+    lib_attr = 'SSL_AD_' + name
+    attr = 'ALERT_DESCRIPTION_' + name
+    if hasattr(lib, lib_attr):
+        globals()[attr] = getattr(lib, lib_attr)
+
+# init open ssl
+lib.SSL_load_error_strings()
+lib.SSL_library_init()
+lib._setup_ssl_threads()
+lib.OpenSSL_add_all_algorithms()
+
+def check_signals():
+    # nothing to do, we are on python level, signals are
+    # checked frequently in the bytecode dispatch loop
+    pass
+
+def _socket_timeout(s):
+    if s is None:
+        return 0.0
+    t = s.gettimeout()
+    if t is None:
+        return -1.0
+    return t
+
+class PasswordInfo(object):
+    callable = None
+    password = None
+    operationerror = None
+    handle = None
+PWINFO_STORAGE = {}
+
+def _Cryptography_pem_password_cb(buf, size, rwflag, userdata):
+    pw_info = ffi.from_handle(userdata)
+
+    password = pw_info.password
+
+    if pw_info.callable:
+        try:
+            password = pw_info.callable()
+        except Exception as e:
+            pw_info.operationerror = e
+            return 0
+
+        if not isinstance(password, (str, bytes, bytearray)):
+            pw_info.operationerror = TypeError("password callback must return a string")
+            return 0
+
+    password = _str_to_ffi_buffer(password)
+
+    if (len(password) > size):
+        pw_info.operationerror = ValueError("password cannot be longer than %d bytes" % size)
+        return 0
+
+    ffi.memmove(buf, password, len(password))
+    return len(password)
+
+if lib.Cryptography_STATIC_CALLBACKS:
+    ffi.def_extern(_Cryptography_pem_password_cb)
+    Cryptography_pem_password_cb = lib.Cryptography_pem_password_cb
+else:
+    Cryptography_pem_password_cb = ffi.callback("int(char*,int,int,void*)")(_Cryptography_pem_password_cb)
+
+if hasattr(time, 'monotonic'):
+    def _monotonic_clock():
+        return time.monotonic()
+else:
+    def _monotonic_clock():
+        return time.clock_gettime(time.CLOCK_MONOTONIC)
+
+def _ssl_select(sock, writing, timeout):
+    if HAVE_POLL:
+        p = poll()
+
+    # Nothing to do unless we're in timeout mode (not non-blocking)
+    if sock is None or timeout == 0:
+        return SOCKET_IS_NONBLOCKING
+    elif timeout < 0:
+        t = _socket_timeout(sock)
+        if t > 0:
+            return SOCKET_HAS_TIMED_OUT
+        else:
+            return SOCKET_IS_BLOCKING
+
+    # Guard against closed socket
+    if sock.fileno() < 0:
+        return SOCKET_HAS_BEEN_CLOSED
+
+    # Prefer poll, if available, since you can poll() any fd
+    # which can't be done with select().
+    if HAVE_POLL:
+        p.register(sock.fileno(), POLLOUT if writing else POLLIN)
+
+        rc = len(p.poll(timeout * 1000.0))
+    else:
+        # currently disabled, see HAVE_POLL
+        fd = sock.fileno()
+        #if (!_PyIsSelectable_fd(s->sock_fd))
+        #    return SOCKET_TOO_LARGE_FOR_SELECT;
+        if writing:
+            rr, wr, xr = select([],[fd],[], timeout)
+        else:
+            rr, wr, xr = select([fd],[],[], timeout)
+        rc = len(rr) + len(wr)
+    if rc != 0:
+        return SOCKET_OPERATION_OK
+    return SOCKET_HAS_TIMED_OUT
+
+SOCKET_IS_NONBLOCKING = 0
+SOCKET_IS_BLOCKING = 1
+SOCKET_HAS_TIMED_OUT = 2
+SOCKET_HAS_BEEN_CLOSED = 3
+SOCKET_TOO_LARGE_FOR_SELECT = 4
+SOCKET_OPERATION_OK = 5
+
+class _SSLSocket(object):
+
+    @staticmethod
+    def _new__ssl_socket(sslctx, sock, socket_type, server_hostname, inbio, outbio):
+        self = _SSLSocket(sslctx)
+        ctx = sslctx.ctx
+
+        if server_hostname:
+            self.server_hostname = server_hostname.decode('idna', 'strict')
+
+        lib.ERR_clear_error()
+        self.ssl = ssl = ffi.gc(lib.SSL_new(ctx), lib.SSL_free)
+
+        self._app_data_handle = ffi.new_handle(self)
+        lib.SSL_set_app_data(ssl, ffi.cast("char*", self._app_data_handle))
+        if sock:
+            lib.SSL_set_fd(ssl, sock.fileno())
+        else:
+            # BIOs are reference counted and SSL_set_bio borrows our reference.
+            # To prevent a double free in memory_bio_dealloc() we need to take an
+            # extra reference here.
+            lib.BIO_up_ref(inbio.bio);
+            lib.BIO_up_ref(outbio.bio);
+            lib.SSL_set_bio(self.ssl, inbio.bio, outbio.bio)
+
+        mode = lib.SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER
+        if lib.SSL_MODE_AUTO_RETRY:
+            mode |= lib.SSL_MODE_AUTO_RETRY
+        lib.SSL_set_mode(ssl, mode)
+
+        if HAS_SNI and self.server_hostname:
+            name = _str_to_ffi_buffer(self.server_hostname)
+            lib.SSL_set_tlsext_host_name(ssl, name)
+
+
+        # If the socket is in non-blocking mode or timeout mode, set the BIO
+        # to non-blocking mode (blocking is the default)
+        #
+        timeout = _socket_timeout(sock)
+        if sock and timeout >= 0:
+            lib.BIO_set_nbio(lib.SSL_get_rbio(ssl), 1)
+            lib.BIO_set_nbio(lib.SSL_get_wbio(ssl), 1)
+
+        if socket_type == SSL_CLIENT:
+            lib.SSL_set_connect_state(ssl)
+        else:
+            lib.SSL_set_accept_state(ssl)
+        self.socket_type = socket_type
+
+        if sock:
+            self.socket = weakref.ref(sock)
+
+        return self
+
+    def __init__(self, sslctx):
+        self.ctx = sslctx
+        self.peer_cert = ffi.NULL
+        self.ssl = ffi.NULL
+        self.shutdown_seen_zero = 0
+        self.handshake_done = 0
+        self._owner = None
+        self.server_hostname = None
+        self.socket = None
+
+    @property
+    def owner(self):
+        if self._owner is None:
+            return None
+        return self._owner()
+
+    @owner.setter
+    def owner(self, value):
+        if value is None:
+            self._owner = None
+        self._owner = weakref.ref(value)
+
+    @property
+    def context(self):
+        return self.ctx
+
+    @context.setter
+    def context(self, value):
+        if isinstance(value, _SSLContext):
+            if not HAS_SNI:
+                raise NotImplementedError("setting a socket's "
+                        "context is not supported by your OpenSSL library")
+            self.ctx = value
+            lib.SSL_set_SSL_CTX(self.ssl, self.ctx.ctx);
+        else:
+            raise TypeError("The value must be a SSLContext")
+
+    @property
+    def server_side(self):
+        return self.socket_type == SSL_SERVER
+
+    def do_handshake(self):
+        sock = self.get_socket_or_connection_gone()
+        ssl = self.ssl
+        timeout = _socket_timeout(sock)
+        if sock:
+            nonblocking = timeout >= 0
+            lib.BIO_set_nbio(lib.SSL_get_rbio(ssl), nonblocking)
+            lib.BIO_set_nbio(lib.SSL_get_wbio(ssl), nonblocking)
+
+        has_timeout = timeout > 0
+        deadline = -1
+        if has_timeout:
+            deadline = _monotonic_clock() + timeout;
+        # Actually negotiate SSL connection
+        # XXX If SSL_do_handshake() returns 0, it's also a failure.
+        while True:
+            # allow threads
+            ret = lib.SSL_do_handshake(ssl)
+            err = lib.SSL_get_error(ssl, ret)
+            # end allow threads
+
+            check_signals()
+
+            if has_timeout:
+                # REIVIEW monotonic clock?
+                timeout = deadline - _monotonic_clock()
+
+            if err == SSL_ERROR_WANT_READ:
+                sockstate = _ssl_select(sock, 0, timeout)
+            elif err == SSL_ERROR_WANT_WRITE:
+                sockstate = _ssl_select(sock, 1, timeout)
+            else:
+                sockstate = SOCKET_OPERATION_OK
+
+            if sockstate == SOCKET_HAS_TIMED_OUT:
+                raise socket.timeout("The handshake operation timed out")
+            elif sockstate == SOCKET_HAS_BEEN_CLOSED:
+                raise SSLError("Underlying socket has been closed.")
+            elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT:
+                raise SSLError("Underlying socket too large for select().")
+            elif sockstate == SOCKET_IS_NONBLOCKING:
+                break
+            if not (err == SSL_ERROR_WANT_READ or err == SSL_ERROR_WANT_WRITE):
+                break
+        if ret < 1:
+            raise pyssl_error(self, ret)
+
+        peer_cert = lib.SSL_get_peer_certificate(ssl)
+        if peer_cert != ffi.NULL:
+            peer_cert = ffi.gc(peer_cert, lib.X509_free)
+        self.peer_cert = peer_cert
+
+        self.handshake_done = 1
+        return None
+
+    def peer_certificate(self, binary_mode):
+        if not self.handshake_done:
+            raise ValueError("handshake not done yet")
+        if self.peer_cert == ffi.NULL:
+            return None
+
+        if binary_mode:
+            # return cert in DER-encoded format
+            return _certificate_to_der(self.peer_cert)
+        else:
+            verification = lib.SSL_CTX_get_verify_mode(lib.SSL_get_SSL_CTX(self.ssl))
+            if (verification & lib.SSL_VERIFY_PEER) == 0:
+                return {}
+            else:
+                return _decode_certificate(self.peer_cert)
+
+    def write(self, bytestring):
+        deadline = 0
+        b = _str_to_ffi_buffer(bytestring)
+        sock = self.get_socket_or_connection_gone()
+        ssl = self.ssl
+
+        if len(b) > sys.maxsize:
+            raise OverflowError("string longer than %d bytes" % sys.maxsize)
+
+        timeout = _socket_timeout(sock)
+        if sock:
+            nonblocking = timeout >= 0
+            lib.BIO_set_nbio(lib.SSL_get_rbio(ssl), nonblocking)
+            lib.BIO_set_nbio(lib.SSL_get_wbio(ssl), nonblocking)
+
+
+        has_timeout = timeout > 0
+        if has_timeout:
+            deadline = _monotonic_clock() + timeout
+
+        sockstate = _ssl_select(sock, 1, timeout)
+        if sockstate == SOCKET_HAS_TIMED_OUT:
+            raise socket.timeout("The write operation timed out")
+        elif sockstate == SOCKET_HAS_BEEN_CLOSED:
+            raise ssl_error("Underlying socket has been closed.")
+        elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT:
+            raise ssl_error("Underlying socket too large for select().")
+
+        while True:
+            length = lib.SSL_write(self.ssl, b, len(b))
+            err = lib.SSL_get_error(self.ssl, length)
+
+            check_signals()
+
+            if has_timeout:
+                timeout = deadline - _monotonic_clock()
+
+            if err == SSL_ERROR_WANT_READ:
+                sockstate = _ssl_select(sock, 0, timeout)
+            elif err == SSL_ERROR_WANT_WRITE:
+                sockstate = _ssl_select(sock, 1, timeout)
+            else:
+                sockstate = SOCKET_OPERATION_OK
+
+            if sockstate == SOCKET_HAS_TIMED_OUT:
+                raise socket.timeout("The write operation timed out")
+            elif sockstate == SOCKET_HAS_BEEN_CLOSED:
+                raise ssl_error("Underlying socket has been closed.")
+            elif sockstate == SOCKET_IS_NONBLOCKING:
+                break
+            if not (err == SSL_ERROR_WANT_READ or err == SSL_ERROR_WANT_WRITE):
+                break
+
+        if length > 0:
+            return length
+        else:
+            raise pyssl_error(self, length)
+
+    def read(self, length, buffer_into=None):
+        ssl = self.ssl
+
+        if length < 0 and buffer_into is None:
+            raise ValueError("size should not be negative")
+
+        sock = self.get_socket_or_connection_gone()
+
+        if buffer_into is None:
+            dest = ffi.new("char[]", length)
+            if length == 0:
+                return b""
+            mem = dest
+        else:
+            mem = ffi.from_buffer(buffer_into)
+            if length <= 0 or length > len(buffer_into):
+                length = len(buffer_into)
+                if length > sys.maxsize:
+                    raise OverflowError("maximum length can't fit in a C 'int'")
+                if len(buffer_into) == 0:
+                    return 0
+
+        if sock:
+            timeout = _socket_timeout(sock)
+            nonblocking = timeout >= 0
+            lib.BIO_set_nbio(lib.SSL_get_rbio(ssl), nonblocking)
+            lib.BIO_set_nbio(lib.SSL_get_wbio(ssl), nonblocking)
+
+        deadline = 0
+        timeout = _socket_timeout(sock)
+        has_timeout = timeout > 0
+        if has_timeout:
+            deadline = _monotonic_clock() + timeout
+
+        shutdown = False
+        while True:
+            count = lib.SSL_read(self.ssl, mem, length);
+            err = lib.SSL_get_error(self.ssl, count);
+
+            check_signals()
+
+            if has_timeout:
+                timeout = deadline - _monotonic_clock()
+
+            if err == SSL_ERROR_WANT_READ:
+                sockstate = _ssl_select(sock, 0, timeout)
+            elif err == SSL_ERROR_WANT_WRITE:
+                sockstate = _ssl_select(sock, 1, timeout)
+            elif err == SSL_ERROR_ZERO_RETURN and \
+                 lib.SSL_get_shutdown(self.ssl) == lib.SSL_RECEIVED_SHUTDOWN:
+                shutdown = True
+                break;
+            else:
+                sockstate = SOCKET_OPERATION_OK
+
+            if sockstate == SOCKET_HAS_TIMED_OUT:
+                raise socket.timeout("The read operation timed out")
+            elif sockstate == SOCKET_IS_NONBLOCKING:
+                break
+            if not (err == SSL_ERROR_WANT_READ or err == SSL_ERROR_WANT_WRITE):
+                break
+
+        if count <= 0 and not shutdown:
+            raise pyssl_error(self, count)
+
+        if not buffer_into:
+            return _bytes_with_len(dest, count)
+
+        return count
+
+    if HAS_ALPN:
+        def selected_alpn_protocol(self):
+            out = ffi.new("const unsigned char **")
+            outlen = ffi.new("unsigned int*")
+
+            lib.SSL_get0_alpn_selected(self.ssl, out, outlen);
+            if out[0] == ffi.NULL:
+                return None
+            return _str_with_len(out[0], outlen[0]);
+
+    def shared_ciphers(self):
+        ciphers = lib.SSL_get_ciphers(self.ssl)
+        if ciphers == ffi.NULL:
+            return None
+        res = []
+        count = lib.sk_SSL_CIPHER_num(ciphers)
+        for i in range(count):
+            tup = cipher_to_tuple(lib.sk_SSL_CIPHER_value(ciphers, i))
+            if not tup:
+                return None
+            res.append(tup)
+        return res
+
+    def cipher(self):
+        if self.ssl == ffi.NULL:
+            return None
+        current = lib.SSL_get_current_cipher(self.ssl)
+        if current == ffi.NULL:
+            return None
+        return cipher_to_tuple(current)
+
+    def compression(self):
+        if not lib.Cryptography_HAS_COMPRESSION or self.ssl == ffi.NULL:
+            return None
+
+        comp_method = lib.SSL_get_current_compression(self.ssl);
+        if comp_method == ffi.NULL: # or lib.SSL_COMP_get_type(comp_method) == lib.NID_undef:
+            return None
+        short_name = lib.SSL_COMP_get_name(comp_method)
+        if short_name == ffi.NULL:
+            return None
+        return _cstr_decode_fs(short_name)
+
+    def version(self):
+        if self.ssl == ffi.NULL:
+            return None
+        version = _str_from_buf(lib.SSL_get_version(self.ssl))
+        if version == "unknown":
+            return None
+        return version
+
+    def get_socket_or_None(self):
+        if self.socket is None:
+            return None
+        return self.socket()
+
+    def get_socket_or_connection_gone(self):
+        """ There are three states:
+            1) self.socket is None (In C that would mean: self->Socket == NULL)
+            2) self.socket() is None (-> The socket is gone)
+            3) self.socket() is not None
+            This method returns True if there is not weakref object allocated
+        """
+        if self.socket is None:
+            return None
+        sock = self.socket()
+        if not sock:
+            raise ssl_error("Underlying socket connection gone", SSL_ERROR_NO_SOCKET)
+        return sock
+
+    def shutdown(self):
+        sock = self.get_socket_or_None()
+        nonblocking = False
+        ssl = self.ssl
+
+        if self.socket is not None:
+            # Guard against closed socket
+            sock = self.socket()
+            if sock is None or sock.fileno() < 0:
+                raise ssl_error("Underlying socket connection gone", SSL_ERROR_NO_SOCKET)
+
+            timeout = _socket_timeout(sock)
+            nonblocking = timeout >= 0
+            if sock and timeout >= 0:
+                lib.BIO_set_nbio(lib.SSL_get_rbio(ssl), nonblocking)
+                lib.BIO_set_nbio(lib.SSL_get_wbio(ssl), nonblocking)
+        else:
+            timeout = 0
+
+        has_timeout = (timeout > 0);
+        if has_timeout:
+            deadline = _monotonic_clock() + timeout;
+
+        zeros = 0
+
+        while True:
+            # Disable read-ahead so that unwrap can work correctly.
+            # Otherwise OpenSSL might read in too much data,
+            # eating clear text data that happens to be
+            # transmitted after the SSL shutdown.
+            # Should be safe to call repeatedly every time this
+            # function is used and the shutdown_seen_zero != 0
+            # condition is met.
+            #
+            if self.shutdown_seen_zero:
+                lib.SSL_set_read_ahead(self.ssl, 0)
+            err = lib.SSL_shutdown(self.ssl)
+
+            # If err == 1, a secure shutdown with SSL_shutdown() is complete
+            if err > 0:
+                break
+            if err == 0:
+                # Don't loop endlessly; instead preserve legacy
+                #   behaviour of trying SSL_shutdown() only twice.
+                #   This looks necessary for OpenSSL < 0.9.8m
+                zeros += 1
+                if zeros > 1:
+                    break
+                # Shutdown was sent, now try receiving
+                self.shutdown_seen_zero = 1
+                continue
+
+            if has_timeout:
+                timeout = deadline - _monotonic_clock()
+
+            # Possibly retry shutdown until timeout or failure
+            ssl_err = lib.SSL_get_error(self.ssl, err)
+            if ssl_err == SSL_ERROR_WANT_READ:
+                sockstate = _ssl_select(sock, 0, timeout)
+            elif ssl_err == SSL_ERROR_WANT_WRITE:
+                sockstate = _ssl_select(sock, 1, timeout)
+            else:
+                break
+
+            if sockstate == SOCKET_HAS_TIMED_OUT:
+                if ssl_err == SSL_ERROR_WANT_READ:
+                    raise socket.timeout("The read operation timed out")
+                else:
+                    raise socket.timeout("The write operation timed out")
+            elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT:
+                raise ssl_error("Underlying socket too large for select().")
+            elif sockstate != SOCKET_OPERATION_OK:
+                # Retain the SSL error code
+                break;
+
+        if err < 0:
+            raise pyssl_error(self, err)
+        if sock:
+            return sock
+        else:
+            return None
+
+    def pending(self):
+        count = lib.SSL_pending(self.ssl)
+        if count < 0:
+            raise pyssl_error(self, count)
+        else:
+            return count
+
+    def tls_unique_cb(self):
+        buf = ffi.new("char[]", SSL_CB_MAXLEN)
+
+        if lib.SSL_session_reused(self.ssl) ^ (not self.server_side):
+            # if session is resumed XOR we are the client
+            length = lib.SSL_get_finished(self.ssl, buf, SSL_CB_MAXLEN)
+        else:
+            # if a new session XOR we are the server
+            length = lib.SSL_get_peer_finished(self.ssl, buf, SSL_CB_MAXLEN)
+
+        # It cannot be negative in current OpenSSL version as of July 2011
+        if length == 0:
+            return None
+
+        return _bytes_with_len(buf, length)
+
+    if HAS_NPN:
+        def selected_npn_protocol(self):
+            out = ffi.new("unsigned char**")
+            outlen = ffi.new("unsigned int*")
+            lib.SSL_get0_next_proto_negotiated(self.ssl, out, outlen)
+            if (out[0] == ffi.NULL):
+                return None
+            return _str_with_len(out[0], outlen[0])
+
+    @property
+    def session(self):
+        "Get / set SSLSession."
+        return SSLSession(self)
+
+    @session.setter
+    def session(self, value):
+        if not isinstance(value, SSLSession):
+            raise TypeError("Value is not a SSLSession.")
+        if self.ctx.ctx != value._ctx.ctx:
+            raise ValueError("Session refers to a different SSLContext.")
+        if self.socket_type != SSL_CLIENT:
+            raise ValueError("Cannot set session for server-side SSLSocket.")
+        if self.handshake_done:
+            raise ValueError("Cannot set session after handshake.")
+        if not lib.SSL_set_session(self.ssl, value._session):
+            raise pyssl_error(self, 0)
+
+    @property
+    def session_reused(self):
+        "Was the client session reused during handshake?"
+        return bool(lib.SSL_session_reused(self.ssl))
+
+
+def _fs_decode(name):
+    return name.decode(sys.getfilesystemencoding())
+def _fs_converter(name):
+    """ name must not be None """
+    if isinstance(name, str):
+        return name.encode(sys.getfilesystemencoding())
+    return bytes(name)
+
+
+def cipher_to_tuple(cipher):
+    ccipher_name = lib.SSL_CIPHER_get_name(cipher)
+    if ccipher_name == ffi.NULL:
+        cipher_name = None
+    else:
+        cipher_name = _str_from_buf(ccipher_name)
+
+    ccipher_protocol = lib.SSL_CIPHER_get_version(cipher)
+    if ccipher_protocol == ffi.NULL:
+        cipher_protocol = None
+    else:
+        cipher_protocol = _str_from_buf(ccipher_protocol)
+
+    bits = lib.SSL_CIPHER_get_bits(cipher, ffi.NULL)
+    return (cipher_name, cipher_protocol, bits)
+
+
+class SSLSession(object):
+    def __new__(cls, ssl):
+        self = object.__new__(cls)
+        session = lib.SSL_get1_session(ssl.ssl)
+        if not session:
+            return None
+        self._session = ffi.gc(session, lib.SSL_SESSION_free)
+        self._ctx = ssl.ctx
+        return self
+
+    def __eq__(self, other):
+        if not isinstance(other, SSLSession):
+            return NotImplemented;
+        return self.id == other.id
+
+    @property
+    def id(self):
+        lenp = ffi.new("unsigned int*")
+        id = lib.SSL_SESSION_get_id(self._session, lenp)
+        return ffi.unpack(id, lenp[0])
+
+    @property
+    def time(self):
+        return lib.SSL_SESSION_get_time(self._session)
+
+    @property
+    def timeout(self):
+        return lib.SSL_SESSION_get_timeout(self._session)
+
+    @property
+    def has_ticket(self):
+        return bool(lib.SSL_SESSION_has_ticket(self._session))
+
+    @property
+    def ticket_lifetime_hint(self):
+        return lib.SSL_SESSION_get_ticket_lifetime_hint(self._session)
+
+
+SSL_CTX_STATS_NAMES = """
+    number connect connect_good connect_renegotiate accept accept_good
+    accept_renegotiate hits misses timeouts cache_full""".split()
+SSL_CTX_STATS = []
+for name in SSL_CTX_STATS_NAMES:
+    attr = 'SSL_CTX_sess_'+name
+    assert hasattr(lib, attr)
+    SSL_CTX_STATS.append((name, getattr(lib, attr)))
+
+class _SSLContext(object):
+    __slots__ = ('ctx', '_check_hostname', 'servername_callback',
+                 'alpn_protocols', '_alpn_protocols_handle',
+                 'npn_protocols', 'set_hostname',
+                 '_set_hostname_handle', '_npn_protocols_handle')
+
+    def __new__(cls, protocol):
+        self = object.__new__(cls)
+        self.ctx = ffi.NULL
+        if protocol == PROTOCOL_TLSv1:
+            method = lib.TLSv1_method()
+        elif lib.Cryptography_HAS_TLSv1_2 and protocol == PROTOCOL_TLSv1_1:
+            method = lib.TLSv1_1_method()
+        elif lib.Cryptography_HAS_TLSv1_2 and protocol == PROTOCOL_TLSv1_2 :
+            method = lib.TLSv1_2_method()
+        elif protocol == PROTOCOL_SSLv3 and lib.Cryptography_HAS_SSL3_METHOD:
+            method = lib.SSLv3_method()
+        elif lib.Cryptography_HAS_SSL2 and protocol == PROTOCOL_SSLv2:
+            method = lib.SSLv2_method()
+        elif protocol == PROTOCOL_SSLv23:
+            method = lib.SSLv23_method()
+        elif protocol == PROTOCOL_TLS_CLIENT:
+            method = lib.SSLv23_client_method()
+        elif protocol == PROTOCOL_TLS_SERVER:
+            method = lib.SSLv23_server_method()
+        else:
+            raise ValueError("invalid protocol version")
+
+        ctx = lib.SSL_CTX_new(method)
+        if ctx == ffi.NULL:
+            raise ssl_error("failed to allocate SSL context")
+        self.ctx = ffi.gc(lib.SSL_CTX_new(method), lib.SSL_CTX_free)
+
+        # Don't check host name by default
+        self._check_hostname = False
+        if protocol == PROTOCOL_TLS_CLIENT:
+            self._check_hostname = True
+            self.verify_mode = CERT_REQUIRED
+        else:
+            self._check_hostname = False
+            self.verify_mode = CERT_NONE
+
+        # Defaults
+        options = lib.SSL_OP_ALL & ~lib.SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS
+        if not lib.Cryptography_HAS_SSL2 or protocol != PROTOCOL_SSLv2:
+            options |= lib.SSL_OP_NO_SSLv2
+        if protocol != PROTOCOL_SSLv3:
+            options |= lib.SSL_OP_NO_SSLv3
+        # Minimal security flags for server and client side context.
+        # Client sockets ignore server-side parameters.
+        options |= lib.SSL_OP_NO_COMPRESSION
+        options |= lib.SSL_OP_CIPHER_SERVER_PREFERENCE
+        options |= lib.SSL_OP_SINGLE_DH_USE
+        options |= lib.SSL_OP_SINGLE_ECDH_USE
+        lib.SSL_CTX_set_options(self.ctx, options)
+        lib.SSL_CTX_set_session_id_context(self.ctx, b"Python", len(b"Python"))
+
+        # A bare minimum cipher list without completely broken cipher suites.
+        # It's far from perfect but gives users a better head start.
+        if lib.Cryptography_HAS_SSL2 and protocol == PROTOCOL_SSLv2:
+            # SSLv2 needs MD5
+            default_ciphers = b"HIGH:!aNULL:!eNULL"
+        else:
+            default_ciphers = b"HIGH:!aNULL:!eNULL:!MD5"
+        if not lib.SSL_CTX_set_cipher_list(ctx, default_ciphers):
+            lib.ERR_clear_error()
+            raise SSLError("No cipher can be selected.")
+
+        if HAS_ECDH:
+            # Allow automatic ECDH curve selection (on
+            # OpenSSL 1.0.2+), or use prime256v1 by default.
+            # This is Apache mod_ssl's initialization
+            # policy, so we should be safe.
+            if lib.Cryptography_HAS_SET_ECDH_AUTO:
+                lib.SSL_CTX_set_ecdh_auto(self.ctx, 1)
+            else:
+                key = lib.EC_KEY_new_by_curve_name(lib.NID_X9_62_prime256v1)
+                lib.SSL_CTX_set_tmp_ecdh(self.ctx, key)
+                lib.EC_KEY_free(key)
+        if lib.Cryptography_HAS_X509_V_FLAG_TRUSTED_FIRST:
+            store = lib.SSL_CTX_get_cert_store(self.ctx)
+            lib.X509_STORE_set_flags(store, lib.X509_V_FLAG_TRUSTED_FIRST)
+        return self
+
+    @property
+    def options(self):
+        return lib.SSL_CTX_get_options(self.ctx)
+
+    @options.setter
+    def options(self, value):
+        new_opts = int(value)
+        opts = lib.SSL_CTX_get_options(self.ctx)
+        clear = opts & ~new_opts
+        set = ~opts & new_opts
+        if clear:
+            if lib.Cryptography_HAS_SSL_CTX_CLEAR_OPTIONS:
+                lib.SSL_CTX_clear_options(self.ctx, clear)
+            else:
+                raise ValueError("can't clear options before OpenSSL 0.9.8m")
+        if set:
+            lib.SSL_CTX_set_options(self.ctx, set)
+
+    @property
+    def verify_mode(self):
+        mode = lib.SSL_CTX_get_verify_mode(self.ctx)
+        if mode == lib.SSL_VERIFY_NONE:
+            return CERT_NONE
+        elif mode == lib.SSL_VERIFY_PEER:
+            return CERT_OPTIONAL
+        elif mode == lib.SSL_VERIFY_PEER | lib.SSL_VERIFY_FAIL_IF_NO_PEER_CERT:
+            return CERT_REQUIRED
+        raise ssl_error("invalid return value from SSL_CTX_get_verify_mode")
+
+    @verify_mode.setter
+    def verify_mode(self, value):
+        n = int(value)
+        if n == CERT_NONE:
+            mode = lib.SSL_VERIFY_NONE
+        elif n == CERT_OPTIONAL:
+            mode = lib.SSL_VERIFY_PEER
+        elif n == CERT_REQUIRED:
+            mode = lib.SSL_VERIFY_PEER | lib.SSL_VERIFY_FAIL_IF_NO_PEER_CERT
+        else:
+            raise ValueError("invalid value for verify_mode")
+        if mode == lib.SSL_VERIFY_NONE and self.check_hostname:
+            raise ValueError("Cannot set verify_mode to CERT_NONE when " \
+                             "check_hostname is enabled.")
+        # Keep current verify cb
+        verify_cb = lib.SSL_CTX_get_verify_callback(self.ctx)
+        lib.SSL_CTX_set_verify(self.ctx, mode, verify_cb)
+
+    @property
+    def verify_flags(self):
+        store = lib.SSL_CTX_get_cert_store(self.ctx)
+        param = lib.X509_STORE_get0_param(store)
+        flags = lib.X509_VERIFY_PARAM_get_flags(param)
+        return int(flags)
+
+    @verify_flags.setter
+    def verify_flags(self, value):
+        new_flags = int(value)
+        store = lib.SSL_CTX_get_cert_store(self.ctx);
+        param = lib.X509_STORE_get0_param(store)
+        flags = lib.X509_VERIFY_PARAM_get_flags(param);
+        clear = flags & ~new_flags;
+        set = ~flags & new_flags;
+        if clear:
+            param = lib.X509_STORE_get0_param(store)
+            if not lib.X509_VERIFY_PARAM_clear_flags(param, clear):
+                raise ssl_error(None, 0)
+        if set:
+            param = lib.X509_STORE_get0_param(store)
+            if not lib.X509_VERIFY_PARAM_set_flags(param, set):
+                raise ssl_error(None, 0)
+
+    @property
+    def check_hostname(self):
+        return self._check_hostname
+
+    @check_hostname.setter
+    def check_hostname(self, value):
+        check_hostname = bool(value)
+        if check_hostname and lib.SSL_CTX_get_verify_mode(self.ctx) == lib.SSL_VERIFY_NONE:
+            raise ValueError("check_hostname needs a SSL context with either "
+                             "CERT_OPTIONAL or CERT_REQUIRED")
+        self._check_hostname = check_hostname
+
+    def set_ciphers(self, cipherlist):
+        cipherlistbuf = _str_to_ffi_buffer(cipherlist)
+        ret = lib.SSL_CTX_set_cipher_list(self.ctx, cipherlistbuf)
+        if ret == 0:
+            # Clearing the error queue is necessary on some OpenSSL
+            # versions, otherwise the error will be reported again
+            # when another SSL call is done.
+            lib.ERR_clear_error()
+            raise ssl_error("No cipher can be selected.")
+
+
+    def load_cert_chain(self, certfile, keyfile=None, password=None):
+        if keyfile is None:
+            keyfile = certfile
+        pw_info = PasswordInfo()
+        index = -1
+        if password is not None:
+
+            if callable(password):
+                pw_info.callable = password
+            else:
+                if isinstance(password, (str, bytes, bytearray)):
+                    pw_info.password = password
+                else:
+                    raise TypeError("password should be a string or callable")
+
+            pw_info.handle = ffi.new_handle(pw_info)
+            index = _thread.get_ident()
+            PWINFO_STORAGE[index] = pw_info
+            lib.SSL_CTX_set_default_passwd_cb(self.ctx, Cryptography_pem_password_cb)
+            lib.SSL_CTX_set_default_passwd_cb_userdata(self.ctx, pw_info.handle)
+
+        try:
+            ffi.errno = 0
+            certfilebuf = _str_to_ffi_buffer(certfile)
+            ret = lib.SSL_CTX_use_certificate_chain_file(self.ctx, certfilebuf)
+            if ret != 1:
+                if pw_info.operationerror:
+                    lib.ERR_clear_error()
+                    raise pw_info.operationerror
+                _errno = ffi.errno
+                if _errno:
+                    lib.ERR_clear_error()
+                    raise OSError(_errno, "Error")
+                else:
+                    raise ssl_error(None)
+
+            ffi.errno = 0
+            buf = _str_to_ffi_buffer(keyfile)
+            ret = lib.SSL_CTX_use_PrivateKey_file(self.ctx, buf,
+                                                  lib.SSL_FILETYPE_PEM)
+            if ret != 1:
+                if pw_info.operationerror:
+                    lib.ERR_clear_error()
+                    raise pw_info.operationerror
+                _errno = ffi.errno
+                if _errno:
+                    lib.ERR_clear_error()
+                    raise OSError(_errno, None)
+                else:
+                    raise ssl_error(None)
+
+            ret = lib.SSL_CTX_check_private_key(self.ctx)
+            if ret != 1:
+                raise ssl_error(None)
+        finally:
+            if index >= 0:
+                del PWINFO_STORAGE[index]
+            lib.SSL_CTX_set_default_passwd_cb(self.ctx, ffi.NULL)
+            lib.SSL_CTX_set_default_passwd_cb_userdata(self.ctx, ffi.NULL)
+
+
+    def _wrap_socket(self, sock, server_side, server_hostname=None):
+        if server_hostname:
+            server_hostname = server_hostname.encode('idna')
+        return _SSLSocket._new__ssl_socket(self, sock, server_side,
+                server_hostname, None, None)
+
+    def load_verify_locations(self, cafile=None, capath=None, cadata=None):
+        ffi.errno = 0
+        if cadata is None:
+            ca_file_type = -1
+        else:
+            if not isinstance(cadata, str):
+                ca_file_type = lib.SSL_FILETYPE_ASN1
+            else:
+                ca_file_type = lib.SSL_FILETYPE_PEM
+                try:
+                    cadata = cadata.encode('ascii')
+                except UnicodeEncodeError:
+                    raise TypeError("cadata should be a ASCII string or a bytes-like object")
+        if cafile is None and capath is None and cadata is None:
+            raise TypeError("cafile and capath cannot be both omitted")
+        # load from cadata
+        if cadata is not None:
+            buf = _str_to_ffi_buffer(cadata)
+            self._add_ca_certs(buf, len(buf), ca_file_type)
+
+        # load cafile or capath
+        if cafile or capath:
+            if cafile is None:
+                cafilebuf = ffi.NULL
+            else:
+                cafilebuf = _str_to_ffi_buffer(cafile)
+            if capath is None:
+                capathbuf = ffi.NULL
+            else:
+                capathbuf = _str_to_ffi_buffer(capath)
+            ret = lib.SSL_CTX_load_verify_locations(self.ctx, cafilebuf, capathbuf)
+            if ret != 1:
+                _errno = ffi.errno
+                if _errno:
+                    lib.ERR_clear_error()
+                    raise OSError(_errno, '')
+                else:
+                    raise ssl_error(None)
+
+    def _add_ca_certs(self, data, size, ca_file_type):
+        biobuf = lib.BIO_new_mem_buf(data, size)
+        if biobuf == ffi.NULL:
+            raise ssl_error("Can't allocate buffer")
+        try:
+            store = lib.SSL_CTX_get_cert_store(self.ctx)
+            loaded = 0
+            while True:
+                if ca_file_type == lib.SSL_FILETYPE_ASN1:
+                    cert = lib.d2i_X509_bio(biobuf, ffi.NULL)
+                else:
+                    cert = lib.PEM_read_bio_X509(biobuf, ffi.NULL, ffi.NULL, ffi.NULL)
+                if not cert:
+                    break
+                try:
+                    r = lib.X509_STORE_add_cert(store, cert)
+                finally:
+                    lib.X509_free(cert)
+                if not r:
+                    err = lib.ERR_peek_last_error()
+                    if (lib.ERR_GET_LIB(err) == lib.ERR_LIB_X509 and
+                        lib.ERR_GET_REASON(err) ==
+                        lib.X509_R_CERT_ALREADY_IN_HASH_TABLE):
+                        # cert already in hash table, not an error
+                        lib.ERR_clear_error()
+                    else:
+                        break
+                loaded += 1
+
+            err = lib.ERR_peek_last_error()
+            if (ca_file_type == lib.SSL_FILETYPE_ASN1 and
+                loaded > 0 and
+                lib.ERR_GET_LIB(err) == lib.ERR_LIB_ASN1 and
+                lib.ERR_GET_REASON(err) == lib.ASN1_R_HEADER_TOO_LONG):
+                # EOF ASN1 file, not an error
+                lib.ERR_clear_error()
+            elif (ca_file_type == lib.SSL_FILETYPE_PEM and
+                  loaded > 0 and
+                  lib.ERR_GET_LIB(err) == lib.ERR_LIB_PEM and
+                  lib.ERR_GET_REASON(err) == lib.PEM_R_NO_START_LINE):
+                # EOF PEM file, not an error
+                lib.ERR_clear_error()
+            else:
+                raise ssl_error(None)
+        finally:
+            lib.BIO_free(biobuf)
+
+    def cert_store_stats(self):
+        store = lib.SSL_CTX_get_cert_store(self.ctx)
+        x509 = 0
+        x509_ca = 0
+        crl = 0
+        objs = lib.X509_STORE_get0_objects(store)
+        count = lib.sk_X509_OBJECT_num(objs)
+        for i in range(count):
+            obj = lib.sk_X509_OBJECT_value(objs, i)
+            _type = lib.X509_OBJECT_get_type(obj)
+            if _type == lib.X509_LU_X509:
+                x509 += 1
+                cert = lib.X509_OBJECT_get0_X509(obj)
+                if lib.X509_check_ca(cert):
+                    x509_ca += 1
+            elif _type == lib.X509_LU_CRL:
+                crl += 1
+            else:
+                # Ignore X509_LU_FAIL, X509_LU_RETRY, X509_LU_PKEY.
+                # As far as I can tell they are internal states and never
+                # stored in a cert store
+                pass
+        return {'x509': x509, 'x509_ca': x509_ca, 'crl': crl}
+
+
+    def session_stats(self):
+        stats = {}
+        for name, ssl_func in SSL_CTX_STATS:
+            stats[name] = ssl_func(self.ctx)
+        return stats
+
+    def set_default_verify_paths(self):
+        if not lib.SSL_CTX_set_default_verify_paths(self.ctx):
+            raise ssl_error("")
+
+    def load_dh_params(self, filepath):
+        ffi.errno = 0
+        if filepath is None:
+            raise TypeError("filepath must not be None")
+        buf = _fs_converter(filepath)
+        mode = ffi.new("char[]",b"r")
+        ffi.errno = 0
+        bio = lib.BIO_new_file(buf, mode)
+        if bio == ffi.NULL:
+            _errno = ffi.errno
+            lib.ERR_clear_error()
+            raise OSError(_errno, '')
+        try:
+            dh = lib.PEM_read_bio_DHparams(bio, ffi.NULL, ffi.NULL, ffi.NULL)
+        finally:
+            lib.BIO_free(bio)
+        if dh == ffi.NULL:
+            _errno = ffi.errno
+            if _errno != 0:
+                lib.ERR_clear_error()
+                raise OSError(_errno, '')
+            else:
+                raise ssl_error(None)
+        try:
+            if lib.SSL_CTX_set_tmp_dh(self.ctx, dh) == 0:
+                raise ssl_error(None)
+        finally:
+            lib.DH_free(dh)
+
+    def get_ca_certs(self, binary_form=None):
+        binary_mode = bool(binary_form)
+        _list = []
+        store = lib.SSL_CTX_get_cert_store(self.ctx)
+        objs = lib.X509_STORE_get0_objects(store)
+        count = lib.sk_X509_OBJECT_num(objs)
+        for i in range(count):
+            obj = lib.sk_X509_OBJECT_value(objs, i)
+            _type = lib.X509_OBJECT_get_type(obj)
+            if _type != lib.X509_LU_X509:
+                # not a x509 cert
+                continue
+            # CA for any purpose
+            cert = lib.X509_OBJECT_get0_X509(obj)
+            if not lib.X509_check_ca(cert):
+                continue
+            if binary_mode:
+                _list.append(_certificate_to_der(cert))
+            else:
+                _list.append(_decode_certificate(cert))
+        return _list
+
+    def set_ecdh_curve(self, name):
+        # needs to be zero terminated
+        if name is None:
+            raise TypeError()
+        buf = _fs_converter(name)
+        nid = lib.OBJ_sn2nid(buf)
+        if nid == 0:
+            raise ValueError("unknown elliptic curve name '%s'" % name)
+        key = lib.EC_KEY_new_by_curve_name(nid)
+        if not key:
+            raise ssl_error(None)
+        try:
+            lib.SSL_CTX_set_tmp_ecdh(self.ctx, key)
+        finally:
+            lib.EC_KEY_free(key)
+
+    def set_servername_callback(self, callback):
+        # cryptography constraint: OPENSSL_NO_TLSEXT will never be set!
+        if not HAS_SNI:
+            raise NotImplementedError("The TLS extension servername callback, "
+                    "SSL_CTX_set_tlsext_servername_callback, "
+                    "is not in the current OpenSSL library.")
+        if callback is None:
+            lib.SSL_CTX_set_tlsext_servername_callback(self.ctx, ffi.NULL)
+            self._set_hostname_handle = None
+            return
+        if not callable(callback):
+            raise TypeError("not a callable object")
+        scb = ServernameCallback(callback, self)
+        self._set_hostname_handle = ffi.new_handle(scb)
+        lib.SSL_CTX_set_tlsext_servername_callback(self.ctx, _servername_callback)
+        lib.SSL_CTX_set_tlsext_servername_arg(self.ctx, self._set_hostname_handle)
+
+    def _set_alpn_protocols(self, protos):
+        if HAS_ALPN:
+            self.alpn_protocols = protocols = ffi.from_buffer(protos)
+            length = len(protocols)
+
+            if lib.SSL_CTX_set_alpn_protos(self.ctx,ffi.cast("unsigned char*", protocols), length):
+                return MemoryError()
+            self._alpn_protocols_handle = handle = ffi.new_handle(self)
+            lib.SSL_CTX_set_alpn_select_cb(self.ctx, select_alpn_callback, handle)
+        else:
+            raise NotImplementedError("The ALPN extension requires OpenSSL 1.0.2 or later.")
+
+    def _set_npn_protocols(self, protos):
+        if HAS_NPN:
+            self.npn_protocols = ffi.from_buffer(protos)
+            handle = ffi.new_handle(self)
+            self._npn_protocols_handle = handle # track a reference to the handle
+            lib.SSL_CTX_set_next_protos_advertised_cb(self.ctx, advertise_npn_callback, handle)
+            lib.SSL_CTX_set_next_proto_select_cb(self.ctx, select_npn_callback, handle)
+        else:
+            raise NotImplementedError("The NPN extension requires OpenSSL 1.0.1 or later.")
+
+    def _wrap_bio(self, incoming, outgoing, server_side, server_hostname):
+        # server_hostname is either None (or absent), or to be encoded
+        # using the idna encoding.
+        hostname = None
+        if server_hostname is not None:
+            hostname = server_hostname.encode("idna")
+
+        sock = _SSLSocket._new__ssl_socket(self, None, server_side, hostname, incoming, outgoing)
+        return sock
+
+
+
+# cryptography constraint: OPENSSL_NO_TLSEXT will never be set!
+if HAS_SNI:
+    @ffi.callback("int(SSL*,int*,void*)")
+    def _servername_callback(s, al, arg):
+        scb = ffi.from_handle(arg)
+        ssl_ctx = scb.ctx
+        servername = lib.SSL_get_servername(s, lib.TLSEXT_NAMETYPE_host_name)
+        set_hostname = scb.callback
+        #ifdef WITH_THREAD
+            # TODO PyGILState_STATE gstate = PyGILState_Ensure();
+        #endif
+
+        if set_hostname is None:
+            #/* remove race condition in this the call back while if removing the
+            # * callback is in progress */
+            #ifdef WITH_THREAD
+                    # TODO PyGILState_Release(gstate);
+            #endif
+            return lib.SSL_TLSEXT_ERR_OK
+
+        ssl = ffi.from_handle(lib.SSL_get_app_data(s))
+        assert isinstance(ssl, _SSLSocket)
+
+        # The servername callback expects an argument that represents the current
+        # SSL connection and that has a .context attribute that can be changed to
+        # identify the requested hostname. Since the official API is the Python
+        # level API we want to pass the callback a Python level object rather than
+        # a _ssl.SSLSocket instance. If there's an "owner" (typically an
+        # SSLObject) that will be passed. Otherwise if there's a socket then that
+        # will be passed. If both do not exist only then the C-level object is
+        # passed.
+        ssl_socket = ssl.owner
+        if not ssl_socket:
+            ssl_socket = ssl.get_socket_or_None()
+
+        if ssl_socket is None:
+            al[0] = lib.SSL_AD_INTERNAL_ERROR
+            return lib.SSL_TLSEXT_ERR_ALERT_FATAL
+
+        if servername == ffi.NULL:
+            try:
+                result = set_hostname(ssl_socket, None, ssl_ctx)
+            except Exception as e:
+                pyerr_write_unraisable(e, set_hostname)
+                al[0] = lib.SSL_AD_HANDSHAKE_FAILURE
+                return lib.SSL_TLSEXT_ERR_ALERT_FATAL
+        else:
+            servername = ffi.string(servername)
+
+            try:
+                servername_idna = servername.decode("idna")
+            except UnicodeDecodeError as e:
+                pyerr_write_unraisable(e, servername)
+
+            try:
+                result = set_hostname(ssl_socket, servername_idna, ssl_ctx)
+            except Exception as e:
+                pyerr_write_unraisable(e, set_hostname)
+                al[0] = lib.SSL_AD_HANDSHAKE_FAILURE
+                return lib.SSL_TLSEXT_ERR_ALERT_FATAL
+
+        if result is not None:
+            # this is just a poor man's emulation:
+            # in CPython this works a bit different, it calls all the way
+            # down from PyLong_AsLong to _PyLong_FromNbInt which raises
+            # a TypeError if there is no nb_int slot filled.
+            try:
+                if isinstance(result, int):
+                    al[0] = result
+                else:
+                    if result is not None:
+                        if hasattr(result,'__int__'):
+                            al[0] = result.__int__()
+                            return lib.SSL_TLSEXT_ERR_ALERT_FATAL
+                    # needed because sys.exec_info is used in pyerr_write_unraisable
+                    raise TypeError("an integer is required (got type %s)" % result)
+            except TypeError as e:
+                pyerr_write_unraisable(e, result)
+                al[0] = lib.SSL_AD_INTERNAL_ERROR
+            return lib.SSL_TLSEXT_ERR_ALERT_FATAL
+        else:
+            # TODO gil state release?
+            return lib.SSL_TLSEXT_ERR_OK
+
+class ServernameCallback(object):
+    def __init__(self, callback, ctx):
+        self.callback = callback
+        self.ctx = ctx
+
+SERVERNAME_CALLBACKS = weakref.WeakValueDictionary()
+
+def _asn1obj2py(obj):
+    nid = lib.OBJ_obj2nid(obj)
+    if nid == lib.NID_undef:
+        raise ValueError("Unknown object")
+    sn = _str_from_buf(lib.OBJ_nid2sn(nid))
+    ln = _str_from_buf(lib.OBJ_nid2ln(nid))
+    buf = ffi.new("char[]", 255)
+    length = lib.OBJ_obj2txt(buf, len(buf), obj, 1)
+    if length < 0:
+        ssl_error(None)
+    if length > 0:
+        return (nid, sn, ln, _str_with_len(buf, length))
+    else:
+        return (nid, sn, ln, None)
+
+def txt2obj(txt, name):
+    _bytes = _str_to_ffi_buffer(txt)
+    is_name = 0 if name else 1
+    obj = lib.OBJ_txt2obj(_bytes, is_name)
+    if obj == ffi.NULL:
+        raise ValueError("unknown object '%s'" % txt)
+    result = _asn1obj2py(obj)
+    lib.ASN1_OBJECT_free(obj)
+    return result
+
+def nid2obj(nid):
+    if nid < lib.NID_undef:
+        raise ValueError("NID must be positive.")
+    obj = lib.OBJ_nid2obj(nid);
+    if obj == ffi.NULL:
+        raise ValueError("unknown NID %i" % nid)
+    result = _asn1obj2py(obj);
+    lib.ASN1_OBJECT_free(obj);
+    return result;
+
+
+class MemoryBIO(object):
+    def __init__(self):
+        bio = lib.BIO_new(lib.BIO_s_mem());
+        if bio == ffi.NULL:
+            raise ssl_error("failed to allocate BIO")
+
+        # Since our BIO is non-blocking an empty read() does not indicate EOF,
+        # just that no data is currently available. The SSL routines should retry
+        # the read, which we can achieve by calling BIO_set_retry_read().
+        lib.BIO_set_retry_read(bio);
+        lib.BIO_set_mem_eof_return(bio, -1);
+
+        self.bio = ffi.gc(bio, lib.BIO_free)
+        self.eof_written = False
+
+    @property
+    def eof(self):
+        """Whether the memory BIO is at EOF."""
+        return lib.BIO_ctrl_pending(self.bio) == 0 and self.eof_written
+
+    def write(self, strlike):
+        INT_MAX = 2**31-1
+        if isinstance(strlike, memoryview):
+            # FIXME pypy must support get_raw_address for
+            # StringBuffer to remove this case!
+            strlike = strlike.tobytes()
+        buf = ffi.from_buffer(strlike)
+        if len(buf) > INT_MAX:
+            raise OverflowError("string longer than %d bytes", INT_MAX)
+
+        if self.eof_written:
+            raise ssl_error("cannot write() after write_eof()")
+        nbytes = lib.BIO_write(self.bio, buf, len(buf));
+        if nbytes < 0:
+            raise ssl_error(None)
+        return nbytes
+
+    def write_eof(self):
+        self.eof_written = True
+        # After an EOF is written, a zero return from read() should be a real EOF
+        # i.e. it should not be retried. Clear the SHOULD_RETRY flag.
+        lib.BIO_clear_retry_flags(self.bio)
+        lib.BIO_set_mem_eof_return(self.bio, 0)
+
+    def read(self, len=-1):
+        count = len
+        avail = lib.BIO_ctrl_pending(self.bio);
+        if count < 0 or count > avail:
+            count = avail;
+
+        buf = ffi.new("char[]", count)
+
+        nbytes = lib.BIO_read(self.bio, buf, count);
+        #  There should never be any short reads but check anyway.
+        if nbytes < count:
+            return b""
+
+        return _bytes_with_len(buf, nbytes)
+
+    @property
+    def pending(self):
+        return lib.BIO_ctrl_pending(self.bio)
+
+
+def RAND_status():
+    return lib.RAND_status()
+
+def _RAND_bytes(count, pseudo):
+    if count < 0:
+        raise ValueError("num must be positive")
+    buf = ffi.new("unsigned char[]", count)
+    if pseudo:
+        # note by reaperhulk, RAND_pseudo_bytes is deprecated in 3.6 already,
+        # it is totally fine to just call RAND_bytes instead
+        ok = lib.RAND_bytes(buf, count)
+        if ok == 1 or ok == 0:
+            _bytes = _bytes_with_len(buf, count)
+            return (_bytes, ok == 1)
+    else:
+        ok = lib.RAND_bytes(buf, count)
+        if ok == 1 or (pseudo and ok == 0):
+            return _bytes_with_len(buf, count)
+    raise ssl_error(None, errcode=lib.ERR_get_error())
+
+def RAND_pseudo_bytes(count):
+    return _RAND_bytes(count, True)
+
+def RAND_bytes(count):
+    return _RAND_bytes(count, False)
+
+def RAND_add(view, entropy):
+    buf = _str_to_ffi_buffer(view)
+    lib.RAND_add(buf, len(buf), entropy)
+
+def get_default_verify_paths():
+
+    ofile_env = _cstr_decode_fs(lib.X509_get_default_cert_file_env())
+    if ofile_env is None:
+        return None
+    ofile = _cstr_decode_fs(lib.X509_get_default_cert_file())
+    if ofile is None:
+        return None
+    odir_env = _cstr_decode_fs(lib.X509_get_default_cert_dir_env())
+    if odir_env is None:
+        return None
+    odir = _cstr_decode_fs(lib.X509_get_default_cert_dir())
+    if odir is None:
+        return odir
+    return (ofile_env, ofile, odir_env, odir);
+
+ at ffi.callback("int(SSL*,unsigned char **,unsigned char *,const unsigned char *,unsigned int,void *)")
+def select_alpn_callback(ssl, out, outlen, client_protocols, client_protocols_len, args):
+    ctx = ffi.from_handle(args)
+    return do_protocol_selection(1, out, outlen,
+                                 ffi.cast("unsigned char*",ctx.alpn_protocols), len(ctx.alpn_protocols),
+                                 client_protocols, client_protocols_len)
+
+if lib.OPENSSL_NPN_NEGOTIATED:
+    @ffi.callback("int(SSL*,unsigned char **,unsigned char *,const unsigned char *,unsigned int,void *)")
+    def select_npn_callback(ssl, out, outlen, server_protocols, server_protocols_len, args):
+        ctx = ffi.from_handle(args)
+        return do_protocol_selection(0, out, outlen, server_protocols, server_protocols_len,
+                                     ffi.cast("unsigned char*",ctx.npn_protocols), len(ctx.npn_protocols))
+
+
+    @ffi.callback("int(SSL*,const unsigned char**, unsigned int*, void*)")
+    def advertise_npn_callback(ssl, data, length, args):
+        ctx = ffi.from_handle(args)
+
+        if not ctx.npn_protocols:
+            data[0] = ffi.new("unsigned char*", b"")
+            length[0] = 0
+        else:
+            data[0] = ffi.cast("unsigned char*",ctx.npn_protocols)
+            length[0] = len(ctx.npn_protocols)
+
+        return lib.SSL_TLSEXT_ERR_OK
+
+
+    def do_protocol_selection(alpn, out, outlen, server_protocols, server_protocols_len,
+                                                 client_protocols, client_protocols_len):
+        if client_protocols == ffi.NULL:
+            client_protocols = b""
+            client_protocols_len = 0
+        if server_protocols == ffi.NULL:
+            server_protocols = ""
+            server_protocols_len = 0
+
+        ret = lib.SSL_select_next_proto(out, outlen,
+                                        server_protocols, server_protocols_len,
+                                        client_protocols, client_protocols_len);
+        if alpn and ret != lib.OPENSSL_NPN_NEGOTIATED:
+            return lib.SSL_TLSEXT_ERR_NOACK
+
+        return lib.SSL_TLSEXT_ERR_OK
+
+if lib.Cryptography_HAS_EGD:
+    def RAND_egd(path):
+        bytecount = lib.RAND_egd_bytes(ffi.from_buffer(path), len(path))
+        if bytecount == -1:
+            raise SSLError("EGD connection failed or EGD did not return "
+                           "enough data to seed the PRNG");
+        return bytecount
+
diff --git a/lib_pypy/_cffi_ssl/_stdssl/certificate.py b/lib_pypy/_cffi_ssl/_stdssl/certificate.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_stdssl/certificate.py
@@ -0,0 +1,342 @@
+import warnings
+import base64
+import textwrap
+from _pypy_openssl import ffi
+from _pypy_openssl import lib
+from _cffi_ssl._stdssl.utility import _string_from_asn1, _str_with_len, _bytes_with_len
+from _cffi_ssl._stdssl.error import ssl_error, pyssl_error
+
+X509_NAME_MAXLEN = 256
+
+def _create_tuple_for_attribute(name, value):
+    buf = ffi.new("char[]", X509_NAME_MAXLEN)
+    length = lib.OBJ_obj2txt(buf, X509_NAME_MAXLEN, name, 0)
+    if length < 0:
+        raise ssl_error(None)
+    name = _str_with_len(buf, length)
+
+    buf_ptr = ffi.new("unsigned char**")
+    length = lib.ASN1_STRING_to_UTF8(buf_ptr, value)
+    if length < 0:
+        raise ssl_error(None)
+    try:
+        value = _str_with_len(buf_ptr[0], length)
+    finally:
+        lib.OPENSSL_free(buf_ptr[0])
+    return (name, value)
+
+def _get_aia_uri(certificate, nid):
+    info = lib.X509_get_ext_d2i(certificate, lib.NID_info_access, ffi.NULL, ffi.NULL)
+    if (info == ffi.NULL):
+        return None;
+    if lib.sk_ACCESS_DESCRIPTION_num(info) == 0:
+        lib.sk_ACCESS_DESCRIPTION_free(info)
+        return None
+
+    lst = []
+    count = lib.sk_ACCESS_DESCRIPTION_num(info)
+    for i in range(count):
+        ad = lib.sk_ACCESS_DESCRIPTION_value(info, i)
+
+        if lib.OBJ_obj2nid(ad.method) != nid or \
+           ad.location.type != lib.GEN_URI:
+            continue
+        uri = ad.location.d.uniformResourceIdentifier
+        ostr = _str_with_len(uri.data, uri.length)
+        lst.append(ostr)
+    lib.sk_ACCESS_DESCRIPTION_free(info)
+
+    # convert to tuple or None
+    if len(lst) == 0: return None
+    return tuple(lst)
+
+def _get_peer_alt_names(certificate):
+    # this code follows the procedure outlined in
+    # OpenSSL's crypto/x509v3/v3_prn.c:X509v3_EXT_print()
+    # function to extract the STACK_OF(GENERAL_NAME),
+    # then iterates through the stack to add the
+    # names.
+    peer_alt_names = []
+
+    if certificate == ffi.NULL:
+        return None
+
+    # get a memory buffer
+    biobuf = lib.BIO_new(lib.BIO_s_mem());
+
+    i = -1
+    while True:
+        i = lib.X509_get_ext_by_NID(certificate, lib.NID_subject_alt_name, i)
+        if i < 0:
+            break
+
+
+        # now decode the altName
+        ext = lib.X509_get_ext(certificate, i);
+        method = lib.X509V3_EXT_get(ext)
+        if method is ffi.NULL:
+            raise ssl_error("No method for internalizing subjectAltName!")
+
+        ext_data = lib.X509_EXTENSION_get_data(ext)
+        ext_data_len = ext_data.length
+        ext_data_value = ffi.new("unsigned char**", ffi.NULL)
+        ext_data_value[0] = ext_data.data
+
+        if method.it != ffi.NULL:
+            names = lib.ASN1_item_d2i(ffi.NULL, ext_data_value, ext_data_len, lib.ASN1_ITEM_ptr(method.it))
+        else:
+            names = method.d2i(ffi.NULL, ext_data_value, ext_data_len)
+
+        names = ffi.cast("GENERAL_NAMES*", names)
+        count = lib.sk_GENERAL_NAME_num(names)
+        for j in range(count):
+            # get a rendering of each name in the set of names
+            name = lib.sk_GENERAL_NAME_value(names, j);
+            _type = name.type
+            if _type == lib.GEN_DIRNAME:
+                # we special-case DirName as a tuple of
+                # tuples of attributes
+                v = _create_tuple_for_X509_NAME(name.d.dirn)
+                peer_alt_names.append(("DirName", v))
+            # GENERAL_NAME_print() doesn't handle NULL bytes in ASN1_string
+            # correctly, CVE-2013-4238
+            elif _type == lib.GEN_EMAIL:
+                v = _string_from_asn1(name.d.rfc822Name)
+                peer_alt_names.append(("email", v))
+            elif _type == lib.GEN_DNS:
+                v = _string_from_asn1(name.d.dNSName)
+                peer_alt_names.append(("DNS", v))
+            elif _type == lib.GEN_URI:
+                v = _string_from_asn1(name.d.uniformResourceIdentifier)
+                peer_alt_names.append(("URI", v))
+            elif _type == lib.GEN_RID:
+                v = "Registered ID"
+                buf = ffi.new("char[2048]")
+
+                length = lib.OBJ_obj2txt(buf, 2047, name.d.rid, 0)
+                if length < 0:
+                    # TODO _setSSLError(NULL, 0, __FILE__, __LINE__);
+                    raise NotImplementedError
+                elif length >= 2048:
+                    v = "<INVALID>"
+                else:
+                    v = _str_with_len(buf, length)
+                peer_alt_names.append(("Registered ID", v))
+            else:
+                # for everything else, we use the OpenSSL print form
+                if _type not in (lib.GEN_OTHERNAME, lib.GEN_X400, \
+                                 lib.GEN_EDIPARTY, lib.GEN_IPADD, lib.GEN_RID):
+                    warnings.warn("Unknown general type %d" % _type, RuntimeWarning)
+                    continue
+                lib.BIO_reset(biobuf);
+                lib.GENERAL_NAME_print(biobuf, name);
+                v = _bio_get_str(biobuf)
+                idx = v.find(":")
+                if idx == -1:
+                    raise ValueError("Invalid value %s", v)
+                peer_alt_names.append((v[:idx], v[idx+1:]))
+
+        free_func_addr = ffi.addressof(lib, "GENERAL_NAME_free")
+        lib.sk_GENERAL_NAME_pop_free(names, free_func_addr);
+    lib.BIO_free(biobuf)
+    if peer_alt_names is not None:
+        return tuple(peer_alt_names)
+    return peer_alt_names
+
+def _create_tuple_for_X509_NAME(xname):
+    dn = []
+    rdn = []
+    rdn_level = -1
+    entry_count = lib.X509_NAME_entry_count(xname);
+    for index_counter in range(entry_count):
+        entry = lib.X509_NAME_get_entry(xname, index_counter);
+
+        # check to see if we've gotten to a new RDN
+        _set = lib.Cryptography_X509_NAME_ENTRY_set(entry)
+        if rdn_level >= 0:
+            if rdn_level != _set:
+                dn.append(tuple(rdn))
+                rdn = []
+        rdn_level = _set
+
+        # now add this attribute to the current RDN
+        name = lib.X509_NAME_ENTRY_get_object(entry);
+        value = lib.X509_NAME_ENTRY_get_data(entry);
+        attr = _create_tuple_for_attribute(name, value);
+        if attr == ffi.NULL:
+            raise NotImplementedError
+        rdn.append(attr)
+
+    # now, there's typically a dangling RDN
+    if rdn and len(rdn) > 0:
+        dn.append(tuple(rdn))
+
+    return tuple(dn)
+
+def _bio_get_str(biobuf):
+    bio_buf = ffi.new("char[]", 2048)
+    length = lib.BIO_gets(biobuf, bio_buf, len(bio_buf)-1)
+    if length < 0:
+        if biobuf: lib.BIO_free(biobuf)
+        raise ssl_error(None)
+    return _str_with_len(bio_buf, length)
+
+def _decode_certificate(certificate):
+    retval = {}
+
+    peer = _create_tuple_for_X509_NAME(lib.X509_get_subject_name(certificate));
+    if not peer:
+        return None
+    retval["subject"] = peer
+
+    issuer = _create_tuple_for_X509_NAME(lib.X509_get_issuer_name(certificate));
+    if not issuer:
+        return None
+    retval["issuer"] = issuer
+
+    version = lib.X509_get_version(certificate) + 1
+    if version == 0:
+        return None
+    retval["version"] = version
+
+    try:
+        biobuf = lib.BIO_new(lib.BIO_s_mem());
+
+        lib.BIO_reset(biobuf);
+        serialNumber = lib.X509_get_serialNumber(certificate);
+        # should not exceed 20 octets, 160 bits, so buf is big enough
+        lib.i2a_ASN1_INTEGER(biobuf, serialNumber)
+        buf = ffi.new("char[]", 2048)
+        length = lib.BIO_gets(biobuf, buf, len(buf)-1)
+        if length < 0:
+            raise ssl_error(None)
+        retval["serialNumber"] = _str_with_len(buf, length)
+
+        lib.BIO_reset(biobuf);
+        notBefore = lib.X509_get_notBefore(certificate);
+        lib.ASN1_TIME_print(biobuf, notBefore);
+        length = lib.BIO_gets(biobuf, buf, len(buf)-1);
+        if length < 0:
+            raise ssl_error(None)
+        retval["notBefore"] = _str_with_len(buf, length)
+
+        lib.BIO_reset(biobuf);
+        notAfter = lib.X509_get_notAfter(certificate);
+        lib.ASN1_TIME_print(biobuf, notAfter);
+        length = lib.BIO_gets(biobuf, buf, len(buf)-1);
+        if length < 0:
+            raise ssl_error(None)
+        retval["notAfter"] = _str_with_len(buf, length)
+
+        # Now look for subjectAltName
+        peer_alt_names = _get_peer_alt_names(certificate);
+        if peer_alt_names is None:
+            return None
+        if len(peer_alt_names) > 0:
+            retval["subjectAltName"] = peer_alt_names
+
+        # Authority Information Access: OCSP URIs
+        obj = _get_aia_uri(certificate, lib.NID_ad_OCSP)
+        if obj:
+            retval["OCSP"] = obj
+
+        obj = _get_aia_uri(certificate, lib.NID_ad_ca_issuers)
+        if obj:
+            retval["caIssuers"] = obj
+
+        # CDP (CRL distribution points)
+        obj = _get_crl_dp(certificate)
+        if obj:
+            retval["crlDistributionPoints"] = obj
+    finally:
+        lib.BIO_free(biobuf)
+
+    return retval
+
+
+def _get_crl_dp(certificate):
+    if lib.OPENSSL_VERSION_NUMBER >= 0x10001000:
+        lib.X509_check_ca(certificate)
+    dps = lib.X509_get_ext_d2i(certificate, lib.NID_crl_distribution_points, ffi.NULL, ffi.NULL)
+    if dps is ffi.NULL:
+        return None
+
+    lst = []
+    count = lib.sk_DIST_POINT_num(dps)
+    for i in range(count):
+        dp = lib.sk_DIST_POINT_value(dps, i);
+        gns = dp.distpoint.name.fullname;
+
+        jcount = lib.sk_GENERAL_NAME_num(gns)
+        for j in range(jcount):
+            gn = lib.sk_GENERAL_NAME_value(gns, j)
+            if gn.type != lib.GEN_URI:
+                continue
+
+            uri = gn.d.uniformResourceIdentifier;
+            ouri = _str_with_len(uri.data, uri.length)
+            lst.append(ouri)
+
+    if lib.OPENSSL_VERSION_NUMBER < 0x10001000:
+        lib.sk_DIST_POINT_free(dps);
+
+    if len(lst) == 0: return None
+    return tuple(lst)
+
+def _test_decode_cert(path):
+    cert = lib.BIO_new(lib.BIO_s_file())
+    if cert is ffi.NULL:
+        lib.BIO_free(cert)
+        raise ssl_error("Can't malloc memory to read file")
+
+    epath = path.encode()
+    if lib.BIO_read_filename(cert, epath) <= 0:
+        lib.BIO_free(cert)
+        raise ssl_error("Can't open file")
+
+    x = lib.PEM_read_bio_X509_AUX(cert, ffi.NULL, ffi.NULL, ffi.NULL)
+    if x is ffi.NULL:
+        ssl_error("Error decoding PEM-encoded file")
+
+    retval = _decode_certificate(x)
+    lib.X509_free(x);
+
+    if cert != ffi.NULL:
+        lib.BIO_free(cert)
+    return retval
+
+PEM_HEADER = "-----BEGIN CERTIFICATE-----"
+PEM_FOOTER = "-----END CERTIFICATE-----"
+
+def PEM_cert_to_DER_cert(pem_cert_string):
+    """Takes a certificate in ASCII PEM format and returns the
+    DER-encoded version of it as a byte sequence"""
+
+    if not pem_cert_string.startswith(PEM_HEADER):
+        raise ValueError("Invalid PEM encoding; must start with %s"
+                         % PEM_HEADER)
+    if not pem_cert_string.strip().endswith(PEM_FOOTER):
+        raise ValueError("Invalid PEM encoding; must end with %s"
+                         % PEM_FOOTER)
+    d = pem_cert_string.strip()[len(PEM_HEADER):-len(PEM_FOOTER)]
+    return base64.decodebytes(d.encode('ASCII', 'strict'))
+
+def DER_cert_to_PEM_cert(der_cert_bytes):
+    """Takes a certificate in binary DER format and returns the
+    PEM version of it as a string."""
+
+    f = str(base64.standard_b64encode(der_cert_bytes), 'ASCII', 'strict')
+    return (PEM_HEADER + '\n' +
+            textwrap.fill(f, 64) + '\n' +
+            PEM_FOOTER + '\n')
+
+def _certificate_to_der(certificate):
+    buf_ptr = ffi.new("unsigned char**")
+    buf_ptr[0] = ffi.NULL
+    length = lib.i2d_X509(certificate, buf_ptr)
+    if length < 0:
+        raise ssl_error(None)
+    try:
+        return _bytes_with_len(ffi.cast("char*",buf_ptr[0]), length)
+    finally:
+        lib.OPENSSL_free(buf_ptr[0])
diff --git a/lib_pypy/_cffi_ssl/_stdssl/error.py b/lib_pypy/_cffi_ssl/_stdssl/error.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_stdssl/error.py
@@ -0,0 +1,202 @@
+import sys
+import os
+import traceback
+from _pypy_openssl import ffi

From pypy.commits at gmail.com  Sat May 11 19:11:45 2019
From: pypy.commits at gmail.com (mattip)
Date: Sat, 11 May 2019 16:11:45 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: remove .gitignore from subdir
Message-ID: <5cd756b1.1c69fb81.629e.1977@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r96600:8cfd74c9502c
Date: 2019-05-10 06:11 -0700
http://bitbucket.org/pypy/pypy/changeset/8cfd74c9502c/

Log:	remove .gitignore from subdir

diff --git a/lib_pypy/_cffi_ssl/.gitignore b/lib_pypy/_cffi_ssl/.gitignore
deleted file mode 100644
--- a/lib_pypy/_cffi_ssl/.gitignore
+++ /dev/null
@@ -1,95 +0,0 @@
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-env/
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-*.egg-info/
-.installed.cfg
-*.egg
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*,cover
-.hypothesis/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# pyenv
-.python-version
-
-# celery beat schedule file
-celerybeat-schedule
-
-# dotenv
-.env
-
-# virtualenv
-.venv/
-venv/
-ENV/
-
-# Spyder project settings
-.spyderproject
-
-# Rope project settings
-.ropeproject
-
-# Vim
-
-*.swp
-*.swo

From pypy.commits at gmail.com  Sat May 11 19:11:46 2019
From: pypy.commits at gmail.com (mattip)
Date: Sat, 11 May 2019 16:11:46 -0700 (PDT)
Subject: [pypy-commit] pypy cffi-libs: hack hack hack
Message-ID: <5cd756b2.1c69fb81.c597.9e72@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cffi-libs
Changeset: r96601:be7db2699e32
Date: 2019-05-10 07:26 -0700
http://bitbucket.org/pypy/pypy/changeset/be7db2699e32/

Log:	hack hack hack

diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -1,7 +1,6 @@
 import sys
 import time
-import _thread
-import socket
+import thread as _thread
 import weakref
 from _pypy_openssl import ffi
 from _pypy_openssl import lib
@@ -21,7 +20,6 @@
         pyerr_write_unraisable)
 from _cffi_ssl._stdssl import error
 from select import select
-from enum import IntEnum as _IntEnum
 
 if sys.platform == 'win32':
     from _cffi_ssl._stdssl.win32_extra import enum_certificates, enum_crls
@@ -30,7 +28,7 @@
     from select import poll, POLLIN, POLLOUT
     HAVE_POLL = True
 
-OPENSSL_VERSION = ffi.string(lib.OPENSSL_VERSION_TEXT).decode('utf-8')
+OPENSSL_VERSION = ffi.string(lib.OPENSSL_VERSION_TEXT)
 OPENSSL_VERSION_NUMBER = lib.OPENSSL_VERSION_NUMBER
 ver = OPENSSL_VERSION_NUMBER
 ver, status = divmod(ver, 16)
@@ -70,6 +68,7 @@
             globals()[name[4:]] = getattr(lib, name)
 
 OP_ALL = lib.SSL_OP_ALL & ~lib.SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS
+OP_NO_SSL_v2 = lib.SSL_OP_NO_SSL_v2
 
 SSL_CLIENT = 0
 SSL_SERVER = 1
@@ -91,8 +90,9 @@
 
 _PROTOCOL_NAMES = (name for name in dir(lib) if name.startswith('PROTOCOL_'))
 
-_IntEnum._convert('_SSLMethod', __name__,
-        lambda name: name.startswith('PROTOCOL_'))
+#from enum import IntEnum as _IntEnum
+#_IntEnum._convert('_SSLMethod', __name__,
+#        lambda name: name.startswith('PROTOCOL_'))
 
 if HAS_TLS_UNIQUE:
     CHANNEL_BINDING_TYPES = ['tls-unique']
@@ -162,13 +162,6 @@
 else:
     Cryptography_pem_password_cb = ffi.callback("int(char*,int,int,void*)")(_Cryptography_pem_password_cb)
 
-if hasattr(time, 'monotonic'):
-    def _monotonic_clock():
-        return time.monotonic()
-else:
-    def _monotonic_clock():
-        return time.clock_gettime(time.CLOCK_MONOTONIC)
-
 def _ssl_select(sock, writing, timeout):
     if HAVE_POLL:
         p = poll()
@@ -217,12 +210,14 @@
 class _SSLSocket(object):
 
     @staticmethod
-    def _new__ssl_socket(sslctx, sock, socket_type, server_hostname, inbio, outbio):
+    def _new__ssl_socket(sslctx, sock, socket_type, server_hostname, ssl_sock):
         self = _SSLSocket(sslctx)
         ctx = sslctx.ctx
 
         if server_hostname:
-            self.server_hostname = server_hostname.decode('idna', 'strict')
+            if isinstance(server_hostname, unicode):
+                server_hostname = server_hostname.encode('idna')
+            self.server_hostname = server_hostname
 
         lib.ERR_clear_error()
         self.ssl = ssl = ffi.gc(lib.SSL_new(ctx), lib.SSL_free)
@@ -235,9 +230,9 @@
             # BIOs are reference counted and SSL_set_bio borrows our reference.
             # To prevent a double free in memory_bio_dealloc() we need to take an
             # extra reference here.
-            lib.BIO_up_ref(inbio.bio);
-            lib.BIO_up_ref(outbio.bio);
-            lib.SSL_set_bio(self.ssl, inbio.bio, outbio.bio)
+            lib.BIO_up_ref(ssl_sock.bio);
+            lib.BIO_up_ref(ssl_sock.bio);
+            lib.SSL_set_bio(self.ssl, ssl_sock.bio, ssl_sock.bio)
 
         mode = lib.SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER
         if lib.SSL_MODE_AUTO_RETRY:
@@ -310,6 +305,9 @@
         return self.socket_type == SSL_SERVER
 
     def do_handshake(self):
+        # delay to prevent circular imports
+        import socket
+
         sock = self.get_socket_or_connection_gone()
         ssl = self.ssl
         timeout = _socket_timeout(sock)
@@ -318,10 +316,6 @@
             lib.BIO_set_nbio(lib.SSL_get_rbio(ssl), nonblocking)
             lib.BIO_set_nbio(lib.SSL_get_wbio(ssl), nonblocking)
 
-        has_timeout = timeout > 0
-        deadline = -1
-        if has_timeout:
-            deadline = _monotonic_clock() + timeout;
         # Actually negotiate SSL connection
         # XXX If SSL_do_handshake() returns 0, it's also a failure.
         while True:
@@ -332,10 +326,6 @@
 
             check_signals()
 
-            if has_timeout:
-                # REIVIEW monotonic clock?
-                timeout = deadline - _monotonic_clock()
-
             if err == SSL_ERROR_WANT_READ:
                 sockstate = _ssl_select(sock, 0, timeout)
             elif err == SSL_ERROR_WANT_WRITE:
@@ -381,6 +371,9 @@
                 return _decode_certificate(self.peer_cert)
 
     def write(self, bytestring):
+        # delay to prevent circular imports
+        import socket
+
         deadline = 0
         b = _str_to_ffi_buffer(bytestring)
         sock = self.get_socket_or_connection_gone()
@@ -395,11 +388,6 @@
             lib.BIO_set_nbio(lib.SSL_get_rbio(ssl), nonblocking)
             lib.BIO_set_nbio(lib.SSL_get_wbio(ssl), nonblocking)
 
-
-        has_timeout = timeout > 0
-        if has_timeout:
-            deadline = _monotonic_clock() + timeout
-
         sockstate = _ssl_select(sock, 1, timeout)
         if sockstate == SOCKET_HAS_TIMED_OUT:
             raise socket.timeout("The write operation timed out")
@@ -414,9 +402,6 @@
 
             check_signals()
 
-            if has_timeout:
-                timeout = deadline - _monotonic_clock()
-
             if err == SSL_ERROR_WANT_READ:
                 sockstate = _ssl_select(sock, 0, timeout)
             elif err == SSL_ERROR_WANT_WRITE:
@@ -439,6 +424,9 @@
             raise pyssl_error(self, length)
 
     def read(self, length, buffer_into=None):
+        # delay to prevent circular imports
+        import socket
+
         ssl = self.ssl
 
         if length < 0 and buffer_into is None:
@@ -468,10 +456,6 @@
 
         deadline = 0
         timeout = _socket_timeout(sock)
-        has_timeout = timeout > 0
-        if has_timeout:
-            deadline = _monotonic_clock() + timeout
-
         shutdown = False
         while True:
             count = lib.SSL_read(self.ssl, mem, length);
@@ -479,9 +463,6 @@
 
             check_signals()
 
-            if has_timeout:
-                timeout = deadline - _monotonic_clock()
-
             if err == SSL_ERROR_WANT_READ:
                 sockstate = _ssl_select(sock, 0, timeout)
             elif err == SSL_ERROR_WANT_WRITE:
@@ -579,6 +560,9 @@
         return sock
 
     def shutdown(self):
+        # delay to prevent circular imports
+        import socket
+
         sock = self.get_socket_or_None()
         nonblocking = False
         ssl = self.ssl
@@ -597,10 +581,6 @@
         else:
             timeout = 0
 
-        has_timeout = (timeout > 0);
-        if has_timeout:
-            deadline = _monotonic_clock() + timeout;
-
         zeros = 0
 
         while True:
@@ -630,9 +610,6 @@
                 self.shutdown_seen_zero = 1
                 continue
 
-            if has_timeout:
-                timeout = deadline - _monotonic_clock()
-
             # Possibly retry shutdown until timeout or failure
             ssl_err = lib.SSL_get_error(self.ssl, err)
             if ssl_err == SSL_ERROR_WANT_READ:
@@ -1028,11 +1005,11 @@
             lib.SSL_CTX_set_default_passwd_cb_userdata(self.ctx, ffi.NULL)
 
 
-    def _wrap_socket(self, sock, server_side, server_hostname=None):
+    def _wrap_socket(self, sock, server_side, server_hostname=None, ssl_sock=None):
         if server_hostname:
             server_hostname = server_hostname.encode('idna')
         return _SSLSocket._new__ssl_socket(self, sock, server_side,
-                server_hostname, None, None)
+                server_hostname, ssl_sock)
 
     def load_verify_locations(self, cafile=None, capath=None, cadata=None):
         ffi.errno = 0
diff --git a/lib_pypy/_cffi_ssl/_stdssl/utility.py b/lib_pypy/_cffi_ssl/_stdssl/utility.py
--- a/lib_pypy/_cffi_ssl/_stdssl/utility.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/utility.py
@@ -19,7 +19,7 @@
     elif isinstance(view, memoryview):
         # NOTE pypy limitation StringBuffer does not allow
         # to get a raw address to the string!
-        view = bytes(view)
+        view = view.tobytes()
     # dont call call ffi.from_buffer(bytes(view)), arguments
     # like ints/bools should result in a TypeError
     return ffi.from_buffer(view)

From pypy.commits at gmail.com  Sat May 11 19:11:48 2019
From: pypy.commits at gmail.com (mattip)
Date: Sat, 11 May 2019 16:11:48 -0700 (PDT)
Subject: [pypy-commit] pypy cffi-libs: add _cffi_src which was "hg ignore"
Message-ID: <5cd756b4.1c69fb81.e39d5.6476@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cffi-libs
Changeset: r96602:92fc282e82f3
Date: 2019-05-10 07:40 -0700
http://bitbucket.org/pypy/pypy/changeset/92fc282e82f3/

Log:	add _cffi_src which was "hg ignore"

diff too long, truncating to 2000 out of 7615 lines

diff --git a/lib_pypy/_cffi_ssl/_cffi_src/__init__.py b/lib_pypy/_cffi_ssl/_cffi_src/__init__.py
new file mode 100644
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/build_commoncrypto.py b/lib_pypy/_cffi_ssl/_cffi_src/build_commoncrypto.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/build_commoncrypto.py
@@ -0,0 +1,33 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+from _cffi_src.utils import build_ffi_for_binding
+
+
+ffi = build_ffi_for_binding(
+    module_name="_commoncrypto",
+    module_prefix="_cffi_src.commoncrypto.",
+    modules=[
+        "cf",
+        "common_digest",
+        "common_hmac",
+        "common_key_derivation",
+        "common_cryptor",
+        "common_symmetric_key_wrap",
+        "seccertificate",
+        "secimport",
+        "secitem",
+        "seckey",
+        "seckeychain",
+        "secpolicy",
+        "sectransform",
+        "sectrust",
+        "secure_transport",
+    ],
+    extra_link_args=[
+        "-framework", "Security", "-framework", "CoreFoundation"
+    ],
+)
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/build_constant_time.py b/lib_pypy/_cffi_ssl/_cffi_src/build_constant_time.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/build_constant_time.py
@@ -0,0 +1,27 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+import os
+
+from _cffi_src.utils import build_ffi, compiler_type, extra_link_args
+
+
+with open(os.path.join(
+    os.path.dirname(__file__), "hazmat_src/constant_time.h"
+)) as f:
+    types = f.read()
+
+with open(os.path.join(
+    os.path.dirname(__file__), "hazmat_src/constant_time.c"
+)) as f:
+    functions = f.read()
+
+ffi = build_ffi(
+    module_name="_constant_time",
+    cdef_source=types,
+    verify_source=functions,
+    extra_link_args=extra_link_args(compiler_type()),
+)
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/build_openssl.py b/lib_pypy/_cffi_ssl/_cffi_src/build_openssl.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/build_openssl.py
@@ -0,0 +1,86 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+import os
+import sys
+
+from _cffi_src.utils import (
+    build_ffi_for_binding, compiler_type, extra_link_args
+)
+
+
+def _get_openssl_libraries(platform):
+    # OpenSSL goes by a different library name on different operating systems.
+    if platform == "darwin":
+        return _osx_libraries(
+            os.environ.get("CRYPTOGRAPHY_OSX_NO_LINK_FLAGS")
+        )
+    elif platform == "win32":
+        if compiler_type() == "msvc":
+            libs = ["libeay32", "ssleay32"]
+        else:
+            libs = ["ssl", "crypto"]
+        return libs + ["advapi32", "crypt32", "gdi32", "user32", "ws2_32"]
+    else:
+        # In some circumstances, the order in which these libs are
+        # specified on the linker command-line is significant;
+        # libssl must come before libcrypto
+        # (http://marc.info/?l=openssl-users&m=135361825921871)
+        return ["ssl", "crypto"]
+
+
+def _osx_libraries(build_static):
+    # For building statically we don't want to pass the -lssl or -lcrypto flags
+    if build_static == "1":
+        return []
+    else:
+        return ["ssl", "crypto"]
+
+
+ffi = build_ffi_for_binding(
+    module_name="_openssl",
+    module_prefix="_cffi_src.openssl.",
+    modules=[
+        # This goes first so we can define some cryptography-wide symbols.
+        "cryptography",
+
+        "aes",
+        "asn1",
+        "bignum",
+        "bio",
+        "cmac",
+        "cms",
+        "conf",
+        "crypto",
+        "dh",
+        "dsa",
+        "ec",
+        "ecdh",
+        "ecdsa",
+        "engine",
+        "err",
+        "evp",
+        "hmac",
+        "nid",
+        "objects",
+        "ocsp",
+        "opensslv",
+        "osrandom_engine",
+        "pem",
+        "pkcs12",
+        "rand",
+        "rsa",
+        "ssl",
+        "x509",
+        "x509name",
+        "x509v3",
+        "x509_vfy",
+        "pkcs7",
+        "callbacks",
+    ],
+    libraries=_get_openssl_libraries(sys.platform),
+    extra_link_args=extra_link_args(compiler_type()),
+)
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/build_padding.py b/lib_pypy/_cffi_ssl/_cffi_src/build_padding.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/build_padding.py
@@ -0,0 +1,27 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+import os
+
+from _cffi_src.utils import build_ffi, compiler_type, extra_link_args
+
+
+with open(os.path.join(
+    os.path.dirname(__file__), "hazmat_src/padding.h"
+)) as f:
+    types = f.read()
+
+with open(os.path.join(
+    os.path.dirname(__file__), "hazmat_src/padding.c"
+)) as f:
+    functions = f.read()
+
+ffi = build_ffi(
+    module_name="_padding",
+    cdef_source=types,
+    verify_source=functions,
+    extra_link_args=extra_link_args(compiler_type()),
+)
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/__init__.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/__init__.py
@@ -0,0 +1,5 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/cf.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/cf.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/cf.py
@@ -0,0 +1,113 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CoreFoundation/CoreFoundation.h>
+"""
+
+TYPES = """
+typedef bool Boolean;
+typedef signed long OSStatus;
+typedef unsigned char UInt8;
+typedef uint32_t UInt32;
+
+typedef const void * CFAllocatorRef;
+const CFAllocatorRef kCFAllocatorDefault;
+typedef ... *CFDataRef;
+typedef signed long long CFIndex;
+typedef ... *CFStringRef;
+typedef ... *CFArrayRef;
+typedef ... *CFMutableArrayRef;
+typedef ... *CFBooleanRef;
+typedef ... *CFErrorRef;
+typedef ... *CFNumberRef;
+typedef ... *CFTypeRef;
+typedef ... *CFDictionaryRef;
+typedef ... *CFMutableDictionaryRef;
+typedef struct {
+    ...;
+} CFDictionaryKeyCallBacks;
+typedef struct {
+    ...;
+} CFDictionaryValueCallBacks;
+typedef struct {
+    ...;
+} CFRange;
+typedef struct {
+    ...;
+} CFArrayCallBacks;
+
+typedef UInt32 CFStringEncoding;
+enum {
+    kCFStringEncodingASCII = 0x0600
+};
+
+enum {
+   kCFNumberSInt8Type = 1,
+   kCFNumberSInt16Type = 2,
+   kCFNumberSInt32Type = 3,
+   kCFNumberSInt64Type = 4,
+   kCFNumberFloat32Type = 5,
+   kCFNumberFloat64Type = 6,
+   kCFNumberCharType = 7,
+   kCFNumberShortType = 8,
+   kCFNumberIntType = 9,
+   kCFNumberLongType = 10,
+   kCFNumberLongLongType = 11,
+   kCFNumberFloatType = 12,
+   kCFNumberDoubleType = 13,
+   kCFNumberCFIndexType = 14,
+   kCFNumberNSIntegerType = 15,
+   kCFNumberCGFloatType = 16,
+   kCFNumberMaxType = 16
+};
+typedef int CFNumberType;
+
+const CFDictionaryKeyCallBacks kCFTypeDictionaryKeyCallBacks;
+const CFDictionaryValueCallBacks kCFTypeDictionaryValueCallBacks;
+
+const CFArrayCallBacks kCFTypeArrayCallBacks;
+
+const CFBooleanRef kCFBooleanTrue;
+const CFBooleanRef kCFBooleanFalse;
+"""
+
+FUNCTIONS = """
+CFDataRef CFDataCreate(CFAllocatorRef, const UInt8 *, CFIndex);
+CFStringRef CFStringCreateWithCString(CFAllocatorRef, const char *,
+                                      CFStringEncoding);
+CFDictionaryRef CFDictionaryCreate(CFAllocatorRef, const void **,
+                                   const void **, CFIndex,
+                                   const CFDictionaryKeyCallBacks *,
+                                   const CFDictionaryValueCallBacks *);
+CFMutableDictionaryRef CFDictionaryCreateMutable(
+    CFAllocatorRef,
+    CFIndex,
+    const CFDictionaryKeyCallBacks *,
+    const CFDictionaryValueCallBacks *
+);
+void CFDictionarySetValue(CFMutableDictionaryRef, const void *, const void *);
+CFIndex CFArrayGetCount(CFArrayRef);
+const void *CFArrayGetValueAtIndex(CFArrayRef, CFIndex);
+CFIndex CFDataGetLength(CFDataRef);
+void CFDataGetBytes(CFDataRef, CFRange, UInt8 *);
+CFRange CFRangeMake(CFIndex, CFIndex);
+void CFShow(CFTypeRef);
+Boolean CFBooleanGetValue(CFBooleanRef);
+CFNumberRef CFNumberCreate(CFAllocatorRef, CFNumberType, const void *);
+void CFRelease(CFTypeRef);
+CFTypeRef CFRetain(CFTypeRef);
+
+CFMutableArrayRef CFArrayCreateMutable(CFAllocatorRef, CFIndex,
+                                       const CFArrayCallBacks *);
+void CFArrayAppendValue(CFMutableArrayRef, const void *);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_cryptor.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_cryptor.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_cryptor.py
@@ -0,0 +1,99 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonCryptor.h>
+"""
+
+TYPES = """
+enum {
+    kCCAlgorithmAES128 = 0,
+    kCCAlgorithmDES,
+    kCCAlgorithm3DES,
+    kCCAlgorithmCAST,
+    kCCAlgorithmRC4,
+    kCCAlgorithmRC2,
+    kCCAlgorithmBlowfish
+};
+typedef uint32_t CCAlgorithm;
+enum {
+    kCCSuccess = 0,
+    kCCParamError = -4300,
+    kCCBufferTooSmall = -4301,
+    kCCMemoryFailure = -4302,
+    kCCAlignmentError = -4303,
+    kCCDecodeError = -4304,
+    kCCUnimplemented = -4305
+};
+typedef int32_t CCCryptorStatus;
+typedef uint32_t CCOptions;
+enum {
+    kCCEncrypt = 0,
+    kCCDecrypt,
+};
+typedef uint32_t CCOperation;
+typedef ... *CCCryptorRef;
+
+enum {
+    kCCModeOptionCTR_LE = 0x0001,
+    kCCModeOptionCTR_BE = 0x0002
+};
+
+typedef uint32_t CCModeOptions;
+
+enum {
+    kCCModeECB = 1,
+    kCCModeCBC = 2,
+    kCCModeCFB = 3,
+    kCCModeCTR = 4,
+    kCCModeF8 = 5,
+    kCCModeLRW = 6,
+    kCCModeOFB = 7,
+    kCCModeXTS = 8,
+    kCCModeRC4 = 9,
+    kCCModeCFB8 = 10,
+    kCCModeGCM = 11
+};
+typedef uint32_t CCMode;
+enum {
+    ccNoPadding = 0,
+    ccPKCS7Padding = 1,
+};
+typedef uint32_t CCPadding;
+"""
+
+FUNCTIONS = """
+CCCryptorStatus CCCryptorCreateWithMode(CCOperation, CCMode, CCAlgorithm,
+                                        CCPadding, const void *, const void *,
+                                        size_t, const void *, size_t, int,
+                                        CCModeOptions, CCCryptorRef *);
+CCCryptorStatus CCCryptorCreate(CCOperation, CCAlgorithm, CCOptions,
+                                const void *, size_t, const void *,
+                                CCCryptorRef *);
+CCCryptorStatus CCCryptorUpdate(CCCryptorRef, const void *, size_t, void *,
+                                size_t, size_t *);
+CCCryptorStatus CCCryptorFinal(CCCryptorRef, void *, size_t, size_t *);
+CCCryptorStatus CCCryptorRelease(CCCryptorRef);
+
+CCCryptorStatus CCCryptorGCMAddIV(CCCryptorRef, const void *, size_t);
+CCCryptorStatus CCCryptorGCMAddAAD(CCCryptorRef, const void *, size_t);
+CCCryptorStatus CCCryptorGCMEncrypt(CCCryptorRef, const void *, size_t,
+                                    void *);
+CCCryptorStatus CCCryptorGCMDecrypt(CCCryptorRef, const void *, size_t,
+                                    void *);
+CCCryptorStatus CCCryptorGCMFinal(CCCryptorRef, const void *, size_t *);
+CCCryptorStatus CCCryptorGCMReset(CCCryptorRef);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+/* Not defined in the public header */
+enum {
+    kCCModeGCM = 11
+};
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_digest.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_digest.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_digest.py
@@ -0,0 +1,58 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonDigest.h>
+"""
+
+TYPES = """
+typedef uint32_t CC_LONG;
+typedef uint64_t CC_LONG64;
+typedef struct CC_MD5state_st {
+    ...;
+} CC_MD5_CTX;
+typedef struct CC_SHA1state_st {
+    ...;
+} CC_SHA1_CTX;
+typedef struct CC_SHA256state_st {
+    ...;
+} CC_SHA256_CTX;
+typedef struct CC_SHA512state_st {
+    ...;
+} CC_SHA512_CTX;
+"""
+
+FUNCTIONS = """
+int CC_MD5_Init(CC_MD5_CTX *);
+int CC_MD5_Update(CC_MD5_CTX *, const void *, CC_LONG);
+int CC_MD5_Final(unsigned char *, CC_MD5_CTX *);
+
+int CC_SHA1_Init(CC_SHA1_CTX *);
+int CC_SHA1_Update(CC_SHA1_CTX *, const void *, CC_LONG);
+int CC_SHA1_Final(unsigned char *, CC_SHA1_CTX *);
+
+int CC_SHA224_Init(CC_SHA256_CTX *);
+int CC_SHA224_Update(CC_SHA256_CTX *, const void *, CC_LONG);
+int CC_SHA224_Final(unsigned char *, CC_SHA256_CTX *);
+
+int CC_SHA256_Init(CC_SHA256_CTX *);
+int CC_SHA256_Update(CC_SHA256_CTX *, const void *, CC_LONG);
+int CC_SHA256_Final(unsigned char *, CC_SHA256_CTX *);
+
+int CC_SHA384_Init(CC_SHA512_CTX *);
+int CC_SHA384_Update(CC_SHA512_CTX *, const void *, CC_LONG);
+int CC_SHA384_Final(unsigned char *, CC_SHA512_CTX *);
+
+int CC_SHA512_Init(CC_SHA512_CTX *);
+int CC_SHA512_Update(CC_SHA512_CTX *, const void *, CC_LONG);
+int CC_SHA512_Final(unsigned char *, CC_SHA512_CTX *);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_hmac.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_hmac.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_hmac.py
@@ -0,0 +1,37 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonHMAC.h>
+"""
+
+TYPES = """
+typedef struct {
+    ...;
+} CCHmacContext;
+enum {
+    kCCHmacAlgSHA1,
+    kCCHmacAlgMD5,
+    kCCHmacAlgSHA256,
+    kCCHmacAlgSHA384,
+    kCCHmacAlgSHA512,
+    kCCHmacAlgSHA224
+};
+typedef uint32_t CCHmacAlgorithm;
+"""
+
+FUNCTIONS = """
+void CCHmacInit(CCHmacContext *, CCHmacAlgorithm, const void *, size_t);
+void CCHmacUpdate(CCHmacContext *, const void *, size_t);
+void CCHmacFinal(CCHmacContext *, void *);
+
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_key_derivation.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_key_derivation.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_key_derivation.py
@@ -0,0 +1,39 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonKeyDerivation.h>
+"""
+
+TYPES = """
+enum {
+    kCCPBKDF2 = 2,
+};
+typedef uint32_t CCPBKDFAlgorithm;
+enum {
+    kCCPRFHmacAlgSHA1 = 1,
+    kCCPRFHmacAlgSHA224 = 2,
+    kCCPRFHmacAlgSHA256 = 3,
+    kCCPRFHmacAlgSHA384 = 4,
+    kCCPRFHmacAlgSHA512 = 5,
+};
+typedef uint32_t CCPseudoRandomAlgorithm;
+typedef unsigned int uint;
+"""
+
+FUNCTIONS = """
+int CCKeyDerivationPBKDF(CCPBKDFAlgorithm, const char *, size_t,
+                         const uint8_t *, size_t, CCPseudoRandomAlgorithm,
+                         uint, uint8_t *, size_t);
+uint CCCalibratePBKDF(CCPBKDFAlgorithm, size_t, size_t,
+                      CCPseudoRandomAlgorithm, size_t, uint32_t);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_symmetric_key_wrap.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_symmetric_key_wrap.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_symmetric_key_wrap.py
@@ -0,0 +1,35 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonSymmetricKeywrap.h>
+"""
+
+TYPES = """
+enum {
+    kCCWRAPAES = 1,
+};
+
+typedef uint32_t CCWrappingAlgorithm;
+"""
+
+FUNCTIONS = """
+int CCSymmetricKeyWrap(CCWrappingAlgorithm, const uint8_t *, const size_t,
+                        const uint8_t *, size_t, const uint8_t *, size_t,
+                        uint8_t *, size_t *);
+int CCSymmetricKeyUnwrap(CCWrappingAlgorithm algorithm, const uint8_t *,
+                         const size_t, const uint8_t *, size_t,
+                         const uint8_t *, size_t, uint8_t *, size_t *);
+size_t CCSymmetricWrappedSize(CCWrappingAlgorithm, size_t);
+size_t CCSymmetricUnwrappedSize(CCWrappingAlgorithm, size_t);
+
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seccertificate.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seccertificate.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seccertificate.py
@@ -0,0 +1,23 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecCertificate.h>
+"""
+
+TYPES = """
+typedef ... *SecCertificateRef;
+"""
+
+FUNCTIONS = """
+SecCertificateRef SecCertificateCreateWithData(CFAllocatorRef, CFDataRef);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secimport.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secimport.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secimport.py
@@ -0,0 +1,86 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecImportExport.h>
+"""
+
+TYPES = """
+typedef ... *SecAccessRef;
+
+CFStringRef kSecImportExportPassphrase;
+CFStringRef kSecImportExportKeychain;
+CFStringRef kSecImportExportAccess;
+
+typedef uint32_t SecExternalItemType;
+enum {
+    kSecItemTypeUnknown,
+    kSecItemTypePrivateKey,
+    kSecItemTypePublicKey,
+    kSecItemTypeSessionKey,
+    kSecItemTypeCertificate,
+    kSecItemTypeAggregate
+};
+
+
+typedef uint32_t SecExternalFormat;
+enum {
+    kSecFormatUnknown = 0,
+    kSecFormatOpenSSL,
+    kSecFormatSSH,
+    kSecFormatBSAFE,
+    kSecFormatRawKey,
+    kSecFormatWrappedPKCS8,
+    kSecFormatWrappedOpenSSL,
+    kSecFormatWrappedSSH,
+    kSecFormatWrappedLSH,
+    kSecFormatX509Cert,
+    kSecFormatPEMSequence,
+    kSecFormatPKCS7,
+    kSecFormatPKCS12,
+    kSecFormatNetscapeCertSequence,
+    kSecFormatSSHv2
+};
+
+typedef uint32_t SecItemImportExportFlags;
+enum {
+    kSecKeyImportOnlyOne        = 0x00000001,
+    kSecKeySecurePassphrase     = 0x00000002,
+    kSecKeyNoAccessControl      = 0x00000004
+};
+typedef uint32_t SecKeyImportExportFlags;
+
+typedef struct {
+    /* for import and export */
+    uint32_t version;
+    SecKeyImportExportFlags  flags;
+    CFTypeRef                passphrase;
+    CFStringRef              alertTitle;
+    CFStringRef              alertPrompt;
+
+    /* for import only */
+    SecAccessRef             accessRef;
+    CFArrayRef               keyUsage;
+
+    CFArrayRef               keyAttributes;
+} SecItemImportExportKeyParameters;
+"""
+
+FUNCTIONS = """
+OSStatus SecItemImport(CFDataRef, CFStringRef, SecExternalFormat *,
+                       SecExternalItemType *, SecItemImportExportFlags,
+                       const SecItemImportExportKeyParameters *,
+                       SecKeychainRef, CFArrayRef *);
+OSStatus SecPKCS12Import(CFDataRef, CFDictionaryRef, CFArrayRef *);
+OSStatus SecItemExport(CFTypeRef, SecExternalFormat, SecItemImportExportFlags,
+                       const SecItemImportExportKeyParameters *, CFDataRef *);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secitem.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secitem.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secitem.py
@@ -0,0 +1,27 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecItem.h>
+"""
+
+TYPES = """
+const CFTypeRef kSecAttrKeyType;
+const CFTypeRef kSecAttrKeySizeInBits;
+const CFTypeRef kSecAttrIsPermanent;
+const CFTypeRef kSecAttrKeyTypeRSA;
+const CFTypeRef kSecAttrKeyTypeDSA;
+const CFTypeRef kSecUseKeychain;
+"""
+
+FUNCTIONS = """
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seckey.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seckey.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seckey.py
@@ -0,0 +1,24 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecKey.h>
+"""
+
+TYPES = """
+typedef ... *SecKeyRef;
+"""
+
+FUNCTIONS = """
+OSStatus SecKeyGeneratePair(CFDictionaryRef, SecKeyRef *, SecKeyRef *);
+size_t SecKeyGetBlockSize(SecKeyRef);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seckeychain.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seckeychain.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seckeychain.py
@@ -0,0 +1,25 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecKeychain.h>
+"""
+
+TYPES = """
+typedef ... *SecKeychainRef;
+"""
+
+FUNCTIONS = """
+OSStatus SecKeychainCreate(const char *, UInt32, const void *, Boolean,
+                           SecAccessRef, SecKeychainRef *);
+OSStatus SecKeychainDelete(SecKeychainRef);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secpolicy.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secpolicy.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secpolicy.py
@@ -0,0 +1,23 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecPolicy.h>
+"""
+
+TYPES = """
+typedef ... *SecPolicyRef;
+"""
+
+FUNCTIONS = """
+SecPolicyRef SecPolicyCreateSSL(Boolean, CFStringRef);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/sectransform.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/sectransform.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/sectransform.py
@@ -0,0 +1,68 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecDigestTransform.h>
+#include <Security/SecSignVerifyTransform.h>
+#include <Security/SecEncryptTransform.h>
+"""
+
+TYPES = """
+typedef ... *SecTransformRef;
+
+CFStringRef kSecImportExportPassphrase;
+CFStringRef kSecImportExportKeychain;
+CFStringRef kSecImportExportAccess;
+
+CFStringRef kSecEncryptionMode;
+CFStringRef kSecEncryptKey;
+CFStringRef kSecIVKey;
+CFStringRef kSecModeCBCKey;
+CFStringRef kSecModeCFBKey;
+CFStringRef kSecModeECBKey;
+CFStringRef kSecModeNoneKey;
+CFStringRef kSecModeOFBKey;
+CFStringRef kSecOAEPEncodingParametersAttributeName;
+CFStringRef kSecPaddingKey;
+CFStringRef kSecPaddingNoneKey;
+CFStringRef kSecPaddingOAEPKey;
+CFStringRef kSecPaddingPKCS1Key;
+CFStringRef kSecPaddingPKCS5Key;
+CFStringRef kSecPaddingPKCS7Key;
+
+const CFStringRef kSecTransformInputAttributeName;
+const CFStringRef kSecTransformOutputAttributeName;
+const CFStringRef kSecTransformDebugAttributeName;
+const CFStringRef kSecTransformTransformName;
+const CFStringRef kSecTransformAbortAttributeName;
+
+CFStringRef kSecInputIsAttributeName;
+CFStringRef kSecInputIsPlainText;
+CFStringRef kSecInputIsDigest;
+CFStringRef kSecInputIsRaw;
+
+const CFStringRef kSecDigestTypeAttribute;
+const CFStringRef kSecDigestLengthAttribute;
+const CFStringRef kSecDigestMD5;
+const CFStringRef kSecDigestSHA1;
+const CFStringRef kSecDigestSHA2;
+"""
+
+FUNCTIONS = """
+Boolean SecTransformSetAttribute(SecTransformRef, CFStringRef, CFTypeRef,
+                                 CFErrorRef *);
+SecTransformRef SecDecryptTransformCreate(SecKeyRef, CFErrorRef *);
+SecTransformRef SecEncryptTransformCreate(SecKeyRef, CFErrorRef *);
+SecTransformRef SecVerifyTransformCreate(SecKeyRef, CFDataRef, CFErrorRef *);
+SecTransformRef SecSignTransformCreate(SecKeyRef, CFErrorRef *) ;
+CFTypeRef SecTransformExecute(SecTransformRef, CFErrorRef *);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/sectrust.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/sectrust.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/sectrust.py
@@ -0,0 +1,39 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecTrust.h>
+"""
+
+TYPES = """
+typedef ... *SecTrustRef;
+typedef uint32_t SecTrustResultType;
+
+enum {
+    kSecTrustResultInvalid,
+    kSecTrustResultProceed,
+    kSecTrustResultDeny,
+    kSecTrustResultUnspecified,
+    kSecTrustResultRecoverableTrustFailure,
+    kSecTrustResultFatalTrustFailure,
+    kSecTrustResultOtherError
+};
+"""
+
+FUNCTIONS = """
+OSStatus SecTrustEvaluate(SecTrustRef, SecTrustResultType *);
+OSStatus SecTrustCopyAnchorCertificates(CFArrayRef *);
+"""
+
+MACROS = """
+/* The first argument changed from CFArrayRef to CFTypeRef in 10.8, so this
+ * has to go here for compatibility.
+ */
+OSStatus SecTrustCreateWithCertificates(CFTypeRef, CFTypeRef, SecTrustRef *);
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secure_transport.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secure_transport.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secure_transport.py
@@ -0,0 +1,308 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecureTransport.h>
+"""
+
+TYPES = """
+typedef ... *SSLContextRef;
+typedef const void *SSLConnectionRef;
+
+typedef enum {
+    kSSLSessionOptionBreakOnServerAuth,
+    kSSLSessionOptionBreakOnCertRequested,
+} SSLSessionOption;
+
+typedef enum {
+    kNeverAuthenticate,
+    kAlwaysAuthenticate,
+    kTryAuthenticate
+} SSLAuthenticate;
+
+typedef enum {
+    kSSLIdle,
+    kSSLHandshake,
+    kSSLConnected,
+    kSSLClosed,
+    kSSLAborted
+} SSLSessionState;
+
+typedef enum {
+    kSSLProtocolUnknown = 0,
+    kSSLProtocol3       = 2,
+    kTLSProtocol1       = 4,
+    /* DEPRECATED on iOS */
+    kSSLProtocol2       = 1,
+    kSSLProtocol3Only   = 3,
+    kTLSProtocol1Only   = 5,
+    kSSLProtocolAll     = 6,
+} SSLProtocol;
+
+typedef UInt32 SSLCipherSuite;
+enum {
+   SSL_NULL_WITH_NULL_NULL =               0x0000,
+   SSL_RSA_WITH_NULL_MD5 =                 0x0001,
+   SSL_RSA_WITH_NULL_SHA =                 0x0002,
+   SSL_RSA_EXPORT_WITH_RC4_40_MD5 =        0x0003,
+   SSL_RSA_WITH_RC4_128_MD5 =              0x0004,
+   SSL_RSA_WITH_RC4_128_SHA =              0x0005,
+   SSL_RSA_EXPORT_WITH_RC2_CBC_40_MD5 =    0x0006,
+   SSL_RSA_WITH_IDEA_CBC_SHA =             0x0007,
+   SSL_RSA_EXPORT_WITH_DES40_CBC_SHA =     0x0008,
+   SSL_RSA_WITH_DES_CBC_SHA =              0x0009,
+   SSL_RSA_WITH_3DES_EDE_CBC_SHA =         0x000A,
+   SSL_DH_DSS_EXPORT_WITH_DES40_CBC_SHA =  0x000B,
+   SSL_DH_DSS_WITH_DES_CBC_SHA =           0x000C,
+   SSL_DH_DSS_WITH_3DES_EDE_CBC_SHA =      0x000D,
+   SSL_DH_RSA_EXPORT_WITH_DES40_CBC_SHA =  0x000E,
+   SSL_DH_RSA_WITH_DES_CBC_SHA =           0x000F,
+   SSL_DH_RSA_WITH_3DES_EDE_CBC_SHA =      0x0010,
+   SSL_DHE_DSS_EXPORT_WITH_DES40_CBC_SHA = 0x0011,
+   SSL_DHE_DSS_WITH_DES_CBC_SHA =          0x0012,
+   SSL_DHE_DSS_WITH_3DES_EDE_CBC_SHA =     0x0013,
+   SSL_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA = 0x0014,
+   SSL_DHE_RSA_WITH_DES_CBC_SHA =          0x0015,
+   SSL_DHE_RSA_WITH_3DES_EDE_CBC_SHA =     0x0016,
+   SSL_DH_anon_EXPORT_WITH_RC4_40_MD5 =    0x0017,
+   SSL_DH_anon_WITH_RC4_128_MD5 =          0x0018,
+   SSL_DH_anon_EXPORT_WITH_DES40_CBC_SHA = 0x0019,
+   SSL_DH_anon_WITH_DES_CBC_SHA =          0x001A,
+   SSL_DH_anon_WITH_3DES_EDE_CBC_SHA =     0x001B,
+   SSL_FORTEZZA_DMS_WITH_NULL_SHA =        0x001C,
+   SSL_FORTEZZA_DMS_WITH_FORTEZZA_CBC_SHA =0x001D,
+
+   /* TLS addenda using AES, per RFC 3268 */
+   TLS_RSA_WITH_AES_128_CBC_SHA      =     0x002F,
+   TLS_DH_DSS_WITH_AES_128_CBC_SHA   =     0x0030,
+   TLS_DH_RSA_WITH_AES_128_CBC_SHA   =     0x0031,
+   TLS_DHE_DSS_WITH_AES_128_CBC_SHA  =     0x0032,
+   TLS_DHE_RSA_WITH_AES_128_CBC_SHA  =     0x0033,
+   TLS_DH_anon_WITH_AES_128_CBC_SHA  =     0x0034,
+   TLS_RSA_WITH_AES_256_CBC_SHA      =     0x0035,
+   TLS_DH_DSS_WITH_AES_256_CBC_SHA   =     0x0036,
+   TLS_DH_RSA_WITH_AES_256_CBC_SHA   =     0x0037,
+   TLS_DHE_DSS_WITH_AES_256_CBC_SHA  =     0x0038,
+   TLS_DHE_RSA_WITH_AES_256_CBC_SHA  =     0x0039,
+   TLS_DH_anon_WITH_AES_256_CBC_SHA  =     0x003A,
+
+   /* ECDSA addenda, RFC 4492 */
+   TLS_ECDH_ECDSA_WITH_NULL_SHA           =    0xC001,
+   TLS_ECDH_ECDSA_WITH_RC4_128_SHA        =    0xC002,
+   TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA   =    0xC003,
+   TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA    =    0xC004,
+   TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA    =    0xC005,
+   TLS_ECDHE_ECDSA_WITH_NULL_SHA          =    0xC006,
+   TLS_ECDHE_ECDSA_WITH_RC4_128_SHA       =    0xC007,
+   TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA  =    0xC008,
+   TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA   =    0xC009,
+   TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA   =    0xC00A,
+   TLS_ECDH_RSA_WITH_NULL_SHA             =    0xC00B,
+   TLS_ECDH_RSA_WITH_RC4_128_SHA          =    0xC00C,
+   TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA     =    0xC00D,
+   TLS_ECDH_RSA_WITH_AES_128_CBC_SHA      =    0xC00E,
+   TLS_ECDH_RSA_WITH_AES_256_CBC_SHA      =    0xC00F,
+   TLS_ECDHE_RSA_WITH_NULL_SHA            =    0xC010,
+   TLS_ECDHE_RSA_WITH_RC4_128_SHA         =    0xC011,
+   TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA    =    0xC012,
+   TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA     =    0xC013,
+   TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA     =    0xC014,
+   TLS_ECDH_anon_WITH_NULL_SHA            =    0xC015,
+   TLS_ECDH_anon_WITH_RC4_128_SHA         =    0xC016,
+   TLS_ECDH_anon_WITH_3DES_EDE_CBC_SHA    =    0xC017,
+   TLS_ECDH_anon_WITH_AES_128_CBC_SHA     =    0xC018,
+   TLS_ECDH_anon_WITH_AES_256_CBC_SHA     =    0xC019,
+
+   /* TLS 1.2 addenda, RFC 5246 */
+   /* Initial state. */
+   TLS_NULL_WITH_NULL_NULL                   = 0x0000,
+
+   /* Server provided RSA certificate for key exchange. */
+   TLS_RSA_WITH_NULL_MD5                     = 0x0001,
+   TLS_RSA_WITH_NULL_SHA                     = 0x0002,
+   TLS_RSA_WITH_RC4_128_MD5                  = 0x0004,
+   TLS_RSA_WITH_RC4_128_SHA                  = 0x0005,
+   TLS_RSA_WITH_3DES_EDE_CBC_SHA             = 0x000A,
+   TLS_RSA_WITH_NULL_SHA256                  = 0x003B,
+   TLS_RSA_WITH_AES_128_CBC_SHA256           = 0x003C,
+   TLS_RSA_WITH_AES_256_CBC_SHA256           = 0x003D,
+
+   /* Server-authenticated (and optionally client-authenticated)
+      Diffie-Hellman. */
+   TLS_DH_DSS_WITH_3DES_EDE_CBC_SHA          = 0x000D,
+   TLS_DH_RSA_WITH_3DES_EDE_CBC_SHA          = 0x0010,
+   TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA         = 0x0013,
+   TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA         = 0x0016,
+   TLS_DH_DSS_WITH_AES_128_CBC_SHA256        = 0x003E,
+   TLS_DH_RSA_WITH_AES_128_CBC_SHA256        = 0x003F,
+   TLS_DHE_DSS_WITH_AES_128_CBC_SHA256       = 0x0040,
+   TLS_DHE_RSA_WITH_AES_128_CBC_SHA256       = 0x0067,
+   TLS_DH_DSS_WITH_AES_256_CBC_SHA256        = 0x0068,
+   TLS_DH_RSA_WITH_AES_256_CBC_SHA256        = 0x0069,
+   TLS_DHE_DSS_WITH_AES_256_CBC_SHA256       = 0x006A,
+   TLS_DHE_RSA_WITH_AES_256_CBC_SHA256       = 0x006B,
+
+   /* Completely anonymous Diffie-Hellman */
+   TLS_DH_anon_WITH_RC4_128_MD5              = 0x0018,
+   TLS_DH_anon_WITH_3DES_EDE_CBC_SHA         = 0x001B,
+   TLS_DH_anon_WITH_AES_128_CBC_SHA256       = 0x006C,
+   TLS_DH_anon_WITH_AES_256_CBC_SHA256       = 0x006D,
+
+   /* Addenda from rfc 5288 AES Galois Counter Mode (GCM) Cipher Suites
+      for TLS. */
+   TLS_RSA_WITH_AES_128_GCM_SHA256           = 0x009C,
+   TLS_RSA_WITH_AES_256_GCM_SHA384           = 0x009D,
+   TLS_DHE_RSA_WITH_AES_128_GCM_SHA256       = 0x009E,
+   TLS_DHE_RSA_WITH_AES_256_GCM_SHA384       = 0x009F,
+   TLS_DH_RSA_WITH_AES_128_GCM_SHA256        = 0x00A0,
+   TLS_DH_RSA_WITH_AES_256_GCM_SHA384        = 0x00A1,
+   TLS_DHE_DSS_WITH_AES_128_GCM_SHA256       = 0x00A2,
+   TLS_DHE_DSS_WITH_AES_256_GCM_SHA384       = 0x00A3,
+   TLS_DH_DSS_WITH_AES_128_GCM_SHA256        = 0x00A4,
+   TLS_DH_DSS_WITH_AES_256_GCM_SHA384        = 0x00A5,
+   TLS_DH_anon_WITH_AES_128_GCM_SHA256       = 0x00A6,
+   TLS_DH_anon_WITH_AES_256_GCM_SHA384       = 0x00A7,
+
+   /* Addenda from rfc 5289  Elliptic Curve Cipher Suites with
+      HMAC SHA-256/384. */
+   TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256   = 0xC023,
+   TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384   = 0xC024,
+   TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256    = 0xC025,
+   TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384    = 0xC026,
+   TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256     = 0xC027,
+   TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384     = 0xC028,
+   TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256      = 0xC029,
+   TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384      = 0xC02A,
+
+   /* Addenda from rfc 5289  Elliptic Curve Cipher Suites with
+      SHA-256/384 and AES Galois Counter Mode (GCM) */
+   TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256   = 0xC02B,
+   TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384   = 0xC02C,
+   TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256    = 0xC02D,
+   TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384    = 0xC02E,
+   TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256     = 0xC02F,
+   TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384     = 0xC030,
+   TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256      = 0xC031,
+   TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384      = 0xC032,
+
+   /* RFC 5746 - Secure Renegotiation */
+   TLS_EMPTY_RENEGOTIATION_INFO_SCSV         = 0x00FF,
+
+   /*
+   * Tags for SSL 2 cipher kinds that are not specified
+   * for SSL 3.
+   */
+   SSL_RSA_WITH_RC2_CBC_MD5 =              0xFF80,
+   SSL_RSA_WITH_IDEA_CBC_MD5 =             0xFF81,
+   SSL_RSA_WITH_DES_CBC_MD5 =              0xFF82,
+   SSL_RSA_WITH_3DES_EDE_CBC_MD5 =         0xFF83,
+   SSL_NO_SUCH_CIPHERSUITE =               0xFFFF
+};
+
+typedef enum {
+    kSSLClientCertNone,
+    kSSLClientCertRequested,
+    kSSLClientCertSent,
+    kSSLClientCertRejected
+} SSLClientCertificateState;
+
+enum {
+    errSSLProtocol              = -9800,
+    errSSLNegotiation           = -9801,
+    errSSLFatalAlert            = -9802,
+    errSSLWouldBlock            = -9803,
+    errSSLSessionNotFound       = -9804,
+    errSSLClosedGraceful        = -9805,
+    errSSLClosedAbort           = -9806,
+    errSSLXCertChainInvalid     = -9807,
+    errSSLBadCert               = -9808,
+    errSSLCrypto                = -9809,
+    errSSLInternal              = -9810,
+    errSSLModuleAttach          = -9811,
+    errSSLUnknownRootCert       = -9812,
+    errSSLNoRootCert            = -9813,
+    errSSLCertExpired           = -9814,
+    errSSLCertNotYetValid       = -9815,
+    errSSLClosedNoNotify        = -9816,
+    errSSLBufferOverflow        = -9817,
+    errSSLBadCipherSuite        = -9818,
+    errSSLPeerUnexpectedMsg     = -9819,
+    errSSLPeerBadRecordMac      = -9820,
+    errSSLPeerDecryptionFail    = -9821,
+    errSSLPeerRecordOverflow    = -9822,
+    errSSLPeerDecompressFail    = -9823,
+    errSSLPeerHandshakeFail     = -9824,
+    errSSLPeerBadCert           = -9825,
+    errSSLPeerUnsupportedCert   = -9826,
+    errSSLPeerCertRevoked       = -9827,
+    errSSLPeerCertExpired       = -9828,
+    errSSLPeerCertUnknown       = -9829,
+    errSSLIllegalParam          = -9830,
+    errSSLPeerUnknownCA         = -9831,
+    errSSLPeerAccessDenied      = -9832,
+    errSSLPeerDecodeError       = -9833,
+    errSSLPeerDecryptError      = -9834,
+    errSSLPeerExportRestriction = -9835,
+    errSSLPeerProtocolVersion   = -9836,
+    errSSLPeerInsufficientSecurity = -9837,
+    errSSLPeerInternalError     = -9838,
+    errSSLPeerUserCancelled     = -9839,
+    errSSLPeerNoRenegotiation   = -9840,
+    errSSLServerAuthCompleted   = -9841,
+    errSSLClientCertRequested   = -9842,
+    errSSLHostNameMismatch      = -9843,
+    errSSLConnectionRefused     = -9844,
+    errSSLDecryptionFail        = -9845,
+    errSSLBadRecordMac          = -9846,
+    errSSLRecordOverflow        = -9847,
+    errSSLBadConfiguration      = -9848,
+    errSSLLast                  = -9849     /* end of range, to be deleted */
+};
+"""
+
+FUNCTIONS = """
+OSStatus SSLSetConnection(SSLContextRef, SSLConnectionRef);
+OSStatus SSLGetConnection(SSLContextRef, SSLConnectionRef *);
+OSStatus SSLSetSessionOption(SSLContextRef, SSLSessionOption, Boolean);
+OSStatus SSLSetClientSideAuthenticate(SSLContextRef, SSLAuthenticate);
+
+OSStatus SSLHandshake(SSLContextRef);
+OSStatus SSLGetSessionState(SSLContextRef, SSLSessionState *);
+OSStatus SSLGetNegotiatedProtocolVersion(SSLContextRef, SSLProtocol *);
+OSStatus SSLSetPeerID(SSLContextRef, const void *, size_t);
+OSStatus SSLGetPeerID(SSLContextRef, const void **, size_t *);
+OSStatus SSLGetBufferedReadSize(SSLContextRef, size_t *);
+OSStatus SSLRead(SSLContextRef, void *, size_t, size_t *);
+OSStatus SSLWrite(SSLContextRef, const void *, size_t, size_t *);
+OSStatus SSLClose(SSLContextRef);
+
+OSStatus SSLGetNumberSupportedCiphers(SSLContextRef, size_t *);
+OSStatus SSLGetSupportedCiphers(SSLContextRef, SSLCipherSuite *, size_t *);
+OSStatus SSLSetEnabledCiphers(SSLContextRef, const SSLCipherSuite *, size_t);
+OSStatus SSLGetNumberEnabledCiphers(SSLContextRef, size_t *);
+OSStatus SSLGetEnabledCiphers(SSLContextRef, SSLCipherSuite *, size_t *);
+OSStatus SSLGetNegotiatedCipher(SSLContextRef, SSLCipherSuite *);
+OSStatus SSLSetDiffieHellmanParams(SSLContextRef, const void *, size_t);
+OSStatus SSLGetDiffieHellmanParams(SSLContextRef, const void **, size_t *);
+
+OSStatus SSLSetCertificateAuthorities(SSLContextRef, CFTypeRef, Boolean);
+OSStatus SSLCopyCertificateAuthorities(SSLContextRef, CFArrayRef *);
+OSStatus SSLCopyDistinguishedNames(SSLContextRef, CFArrayRef *);
+OSStatus SSLSetCertificate(SSLContextRef, CFArrayRef);
+OSStatus SSLGetClientCertificateState(SSLContextRef,
+                                      SSLClientCertificateState *);
+OSStatus SSLCopyPeerTrust(SSLContextRef, SecTrustRef *trust);
+
+OSStatus SSLSetPeerDomainName(SSLContextRef, const char *, size_t);
+OSStatus SSLGetPeerDomainNameLength(SSLContextRef, size_t *);
+OSStatus SSLGetPeerDomainName(SSLContextRef, char *, size_t *);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/constant_time.c b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/constant_time.c
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/constant_time.c
@@ -0,0 +1,22 @@
+// This file is dual licensed under the terms of the Apache License, Version
+// 2.0, and the BSD License. See the LICENSE file in the root of this
+// repository for complete details.
+
+uint8_t Cryptography_constant_time_bytes_eq(uint8_t *a, size_t len_a,
+                                            uint8_t *b, size_t len_b) {
+    size_t i = 0;
+    uint8_t mismatch = 0;
+    if (len_a != len_b) {
+        return 0;
+    }
+    for (i = 0; i < len_a; i++) {
+        mismatch |= a[i] ^ b[i];
+    }
+
+    /* Make sure any bits set are copied to the lowest bit */
+    mismatch |= mismatch >> 4;
+    mismatch |= mismatch >> 2;
+    mismatch |= mismatch >> 1;
+    /* Now check the low bit to see if it's set */
+    return (mismatch & 1) == 0;
+}
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/constant_time.h b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/constant_time.h
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/constant_time.h
@@ -0,0 +1,6 @@
+// This file is dual licensed under the terms of the Apache License, Version
+// 2.0, and the BSD License. See the LICENSE file in the root of this
+// repository for complete details.
+
+uint8_t Cryptography_constant_time_bytes_eq(uint8_t *, size_t, uint8_t *,
+                                            size_t);
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/padding.c b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/padding.c
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/padding.c
@@ -0,0 +1,65 @@
+// This file is dual licensed under the terms of the Apache License, Version
+// 2.0, and the BSD License. See the LICENSE file in the root of this
+// repository for complete details.
+
+/* Returns the value of the input with the most-significant-bit copied to all
+   of the bits. */
+static uint16_t Cryptography_DUPLICATE_MSB_TO_ALL(uint16_t a) {
+    return (1 - (a >> (sizeof(uint16_t) * 8 - 1))) - 1;
+}
+
+/* This returns 0xFFFF if a < b else 0x0000, but does so in a constant time
+   fashion */
+static uint16_t Cryptography_constant_time_lt(uint16_t a, uint16_t b) {
+    a -= b;
+    return Cryptography_DUPLICATE_MSB_TO_ALL(a);
+}
+
+uint8_t Cryptography_check_pkcs7_padding(const uint8_t *data,
+                                         uint16_t block_len) {
+    uint16_t i;
+    uint16_t pad_size = data[block_len - 1];
+    uint16_t mismatch = 0;
+    for (i = 0; i < block_len; i++) {
+        unsigned int mask = Cryptography_constant_time_lt(i, pad_size);
+        uint16_t b = data[block_len - 1 - i];
+        mismatch |= (mask & (pad_size ^ b));
+    }
+
+    /* Check to make sure the pad_size was within the valid range. */
+    mismatch |= ~Cryptography_constant_time_lt(0, pad_size);
+    mismatch |= Cryptography_constant_time_lt(block_len, pad_size);
+
+    /* Make sure any bits set are copied to the lowest bit */
+    mismatch |= mismatch >> 8;
+    mismatch |= mismatch >> 4;
+    mismatch |= mismatch >> 2;
+    mismatch |= mismatch >> 1;
+    /* Now check the low bit to see if it's set */
+    return (mismatch & 1) == 0;
+}
+
+uint8_t Cryptography_check_ansix923_padding(const uint8_t *data,
+                                            uint16_t block_len) {
+    uint16_t i;
+    uint16_t pad_size = data[block_len - 1];
+    uint16_t mismatch = 0;
+    /* Skip the first one with the pad size */
+    for (i = 1; i < block_len; i++) {
+        unsigned int mask = Cryptography_constant_time_lt(i, pad_size);
+        uint16_t b = data[block_len - 1 - i];
+        mismatch |= (mask & b);
+    }
+
+    /* Check to make sure the pad_size was within the valid range. */
+    mismatch |= ~Cryptography_constant_time_lt(0, pad_size);
+    mismatch |= Cryptography_constant_time_lt(block_len, pad_size);
+
+    /* Make sure any bits set are copied to the lowest bit */
+    mismatch |= mismatch >> 8;
+    mismatch |= mismatch >> 4;
+    mismatch |= mismatch >> 2;
+    mismatch |= mismatch >> 1;
+    /* Now check the low bit to see if it's set */
+    return (mismatch & 1) == 0;
+}
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/padding.h b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/padding.h
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/padding.h
@@ -0,0 +1,6 @@
+// This file is dual licensed under the terms of the Apache License, Version
+// 2.0, and the BSD License. See the LICENSE file in the root of this
+// repository for complete details.
+
+uint8_t Cryptography_check_pkcs7_padding(const uint8_t *, uint8_t);
+uint8_t Cryptography_check_ansix923_padding(const uint8_t *, uint8_t);
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/__init__.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/__init__.py
@@ -0,0 +1,5 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/aes.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/aes.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/aes.py
@@ -0,0 +1,50 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <openssl/aes.h>
+"""
+
+TYPES = """
+static const int Cryptography_HAS_AES_WRAP;
+static const int Cryptography_HAS_AES_CTR128_ENCRYPT;
+
+struct aes_key_st {
+    ...;
+};
+typedef struct aes_key_st AES_KEY;
+"""
+
+FUNCTIONS = """
+int AES_set_encrypt_key(const unsigned char *, const int, AES_KEY *);
+int AES_set_decrypt_key(const unsigned char *, const int, AES_KEY *);
+
+int AES_wrap_key(AES_KEY *, const unsigned char *, unsigned char *,
+                 const unsigned char *, unsigned int);
+int AES_unwrap_key(AES_KEY *, const unsigned char *, unsigned char *,
+                   const unsigned char *, unsigned int);
+"""
+
+MACROS = """
+/* The ctr128_encrypt function is only useful in 1.0.0. We can use EVP for
+   this in 1.0.1+. */
+void AES_ctr128_encrypt(const unsigned char *, unsigned char *,
+                        size_t, const AES_KEY *, unsigned char[],
+                        unsigned char[], unsigned int *);
+"""
+
+CUSTOMIZATIONS = """
+static const long Cryptography_HAS_AES_WRAP = 1;
+#if CRYPTOGRAPHY_OPENSSL_110_OR_GREATER && !defined(LIBRESSL_VERSION_NUMBER)
+static const int Cryptography_HAS_AES_CTR128_ENCRYPT = 0;
+void (*AES_ctr128_encrypt)(const unsigned char *, unsigned char *,
+                           size_t, const AES_KEY *,
+                           unsigned char[], unsigned char[],
+                           unsigned int *) = NULL;
+#else
+static const int Cryptography_HAS_AES_CTR128_ENCRYPT = 1;
+#endif
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/asn1.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/asn1.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/asn1.py
@@ -0,0 +1,164 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <openssl/asn1.h>
+"""
+
+TYPES = """
+typedef int... time_t;
+
+typedef int ASN1_BOOLEAN;
+typedef ... ASN1_INTEGER;
+
+struct asn1_string_st {
+    int length;
+    int type;
+    unsigned char *data;
+    long flags;
+};
+
+typedef struct asn1_string_st ASN1_OCTET_STRING;
+typedef struct asn1_string_st ASN1_IA5STRING;
+typedef struct asn1_string_st ASN1_BIT_STRING;
+typedef struct asn1_string_st ASN1_TIME;
+typedef ... ASN1_OBJECT;
+typedef struct asn1_string_st ASN1_STRING;
+typedef struct asn1_string_st ASN1_UTF8STRING;
+typedef ... ASN1_TYPE;
+typedef ... ASN1_GENERALIZEDTIME;
+typedef ... ASN1_ENUMERATED;
+typedef ... ASN1_ITEM;
+typedef ... ASN1_VALUE;
+
+typedef ... ASN1_ITEM_EXP;
+
+typedef ... ASN1_UTCTIME;
+
+static const int V_ASN1_GENERALIZEDTIME;
+
+static const int MBSTRING_FLAG;
+static const int MBSTRING_ASC;
+static const int MBSTRING_BMP;
+static const int MBSTRING_UTF8;
+static const int MBSTRING_UNIV;
+"""
+
+FUNCTIONS = """
+ASN1_OBJECT *ASN1_OBJECT_new(void);
+void ASN1_OBJECT_free(ASN1_OBJECT *);
+
+/*  ASN1 OBJECT IDENTIFIER */
+ASN1_OBJECT *d2i_ASN1_OBJECT(ASN1_OBJECT **, const unsigned char **, long);
+
+/*  ASN1 STRING */
+ASN1_STRING *ASN1_STRING_new(void);
+ASN1_STRING *ASN1_STRING_type_new(int);
+void ASN1_STRING_free(ASN1_STRING *);
+unsigned char *ASN1_STRING_data(ASN1_STRING *);
+int ASN1_STRING_set(ASN1_STRING *, const void *, int);
+
+/*  ASN1 OCTET STRING */
+ASN1_OCTET_STRING *ASN1_OCTET_STRING_new(void);
+void ASN1_OCTET_STRING_free(ASN1_OCTET_STRING *);
+int ASN1_OCTET_STRING_set(ASN1_OCTET_STRING *, const unsigned char *, int);
+
+/* ASN1 IA5STRING */
+ASN1_IA5STRING *ASN1_IA5STRING_new(void);
+
+/*  ASN1 INTEGER */
+ASN1_INTEGER *ASN1_INTEGER_new(void);
+void ASN1_INTEGER_free(ASN1_INTEGER *);
+int ASN1_INTEGER_set(ASN1_INTEGER *, long);
+
+/*  ASN1 TIME */
+ASN1_TIME *ASN1_TIME_new(void);
+void ASN1_TIME_free(ASN1_TIME *);
+ASN1_GENERALIZEDTIME *ASN1_TIME_to_generalizedtime(ASN1_TIME *,
+                                                   ASN1_GENERALIZEDTIME **);
+ASN1_TIME *ASN1_TIME_set(ASN1_TIME *, time_t);
+
+/*  ASN1 UTCTIME */
+ASN1_UTCTIME *ASN1_UTCTIME_new(void);
+void ASN1_UTCTIME_free(ASN1_UTCTIME *);
+int ASN1_UTCTIME_cmp_time_t(const ASN1_UTCTIME *, time_t);
+ASN1_UTCTIME *ASN1_UTCTIME_set(ASN1_UTCTIME *, time_t);
+
+/*  ASN1 GENERALIZEDTIME */
+int ASN1_GENERALIZEDTIME_set_string(ASN1_GENERALIZEDTIME *, const char *);
+ASN1_GENERALIZEDTIME *ASN1_GENERALIZEDTIME_set(ASN1_GENERALIZEDTIME *, time_t);
+void ASN1_GENERALIZEDTIME_free(ASN1_GENERALIZEDTIME *);
+int i2d_ASN1_GENERALIZEDTIME(ASN1_GENERALIZEDTIME *, unsigned char **);
+
+/*  ASN1 ENUMERATED */
+ASN1_ENUMERATED *ASN1_ENUMERATED_new(void);
+void ASN1_ENUMERATED_free(ASN1_ENUMERATED *);
+int ASN1_ENUMERATED_set(ASN1_ENUMERATED *, long);
+int i2d_ASN1_ENUMERATED(ASN1_ENUMERATED *, unsigned char **);
+
+ASN1_VALUE *ASN1_item_d2i(ASN1_VALUE **, const unsigned char **, long,
+                          const ASN1_ITEM *);
+int ASN1_BIT_STRING_set_bit(ASN1_BIT_STRING *, int, int);
+"""
+
+MACROS = """
+/* These became const ASN1_* in 1.1.0 */
+int i2d_ASN1_OBJECT(ASN1_OBJECT *, unsigned char **);
+int ASN1_STRING_type(ASN1_STRING *);
+int ASN1_STRING_to_UTF8(unsigned char **, ASN1_STRING *);
+long ASN1_ENUMERATED_get(ASN1_ENUMERATED *);
+int i2a_ASN1_INTEGER(BIO *, ASN1_INTEGER *);
+
+ASN1_UTF8STRING *ASN1_UTF8STRING_new(void);
+void ASN1_UTF8STRING_free(ASN1_UTF8STRING *);
+
+ASN1_BIT_STRING *ASN1_BIT_STRING_new(void);
+void ASN1_BIT_STRING_free(ASN1_BIT_STRING *);
+int i2d_ASN1_BIT_STRING(ASN1_BIT_STRING *, unsigned char **);
+int i2d_ASN1_OCTET_STRING(ASN1_OCTET_STRING *, unsigned char **);
+int i2d_ASN1_INTEGER(ASN1_INTEGER *, unsigned char **);
+/* This is not a macro, but is const on some versions of OpenSSL */
+int ASN1_BIT_STRING_get_bit(ASN1_BIT_STRING *, int);
+ASN1_TIME *M_ASN1_TIME_dup(void *);
+const ASN1_ITEM *ASN1_ITEM_ptr(ASN1_ITEM_EXP *);
+
+/* These aren't macros these arguments are all const X on openssl > 1.0.x */
+
+int ASN1_TIME_print(BIO *, ASN1_TIME *);
+int ASN1_STRING_length(ASN1_STRING *);
+ASN1_STRING *ASN1_STRING_dup(ASN1_STRING *);
+int ASN1_STRING_cmp(ASN1_STRING *, ASN1_STRING *);
+int ASN1_UTCTIME_print(BIO *, ASN1_UTCTIME *);
+
+ASN1_OCTET_STRING *ASN1_OCTET_STRING_dup(ASN1_OCTET_STRING *);
+int ASN1_OCTET_STRING_cmp(ASN1_OCTET_STRING *, ASN1_OCTET_STRING *);
+
+ASN1_INTEGER *ASN1_INTEGER_dup(ASN1_INTEGER *);
+int ASN1_INTEGER_cmp(ASN1_INTEGER *, ASN1_INTEGER *);
+long ASN1_INTEGER_get(ASN1_INTEGER *);
+
+BIGNUM *ASN1_INTEGER_to_BN(ASN1_INTEGER *, BIGNUM *);
+ASN1_INTEGER *BN_to_ASN1_INTEGER(BIGNUM *, ASN1_INTEGER *);
+
+/* These isn't a macro the arg is const on openssl 1.0.2+ */
+int ASN1_GENERALIZEDTIME_check(ASN1_GENERALIZEDTIME *);
+int ASN1_UTCTIME_check(ASN1_UTCTIME *);
+
+/* Not a macro, const on openssl 1.0 */
+int ASN1_STRING_set_default_mask_asc(char *);
+
+int i2d_ASN1_TYPE(ASN1_TYPE *, unsigned char **);
+ASN1_TYPE *d2i_ASN1_TYPE(ASN1_TYPE **, const unsigned char **, long);
+"""
+
+CUSTOMIZATIONS = """
+/* This macro is removed in 1.1.0. We re-add it if required to support
+   pyOpenSSL versions older than whatever resolves
+   https://github.com/pyca/pyopenssl/issues/431 */
+#if !defined(M_ASN1_TIME_dup)
+#define M_ASN1_TIME_dup(a) (ASN1_TIME *)ASN1_STRING_dup((const ASN1_STRING *)a)
+#endif
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/bignum.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/bignum.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/bignum.py
@@ -0,0 +1,88 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <openssl/bn.h>
+"""
+
+TYPES = """
+typedef ... BN_CTX;
+typedef ... BIGNUM;
+typedef int... BN_ULONG;
+"""
+
+FUNCTIONS = """
+BIGNUM *BN_new(void);
+void BN_free(BIGNUM *);
+
+BN_CTX *BN_CTX_new(void);
+void BN_CTX_free(BN_CTX *);
+
+void BN_CTX_start(BN_CTX *);
+BIGNUM *BN_CTX_get(BN_CTX *);
+void BN_CTX_end(BN_CTX *);
+
+BIGNUM *BN_copy(BIGNUM *, const BIGNUM *);
+BIGNUM *BN_dup(const BIGNUM *);
+
+int BN_set_word(BIGNUM *, BN_ULONG);
+BN_ULONG BN_get_word(const BIGNUM *);
+
+const BIGNUM *BN_value_one(void);
+
+char *BN_bn2hex(const BIGNUM *);
+int BN_hex2bn(BIGNUM **, const char *);
+int BN_dec2bn(BIGNUM **, const char *);
+
+int BN_bn2bin(const BIGNUM *, unsigned char *);
+BIGNUM *BN_bin2bn(const unsigned char *, int, BIGNUM *);
+
+int BN_num_bits(const BIGNUM *);
+
+int BN_cmp(const BIGNUM *, const BIGNUM *);
+int BN_add(BIGNUM *, const BIGNUM *, const BIGNUM *);
+int BN_sub(BIGNUM *, const BIGNUM *, const BIGNUM *);
+int BN_mul(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+int BN_sqr(BIGNUM *, const BIGNUM *, BN_CTX *);
+int BN_div(BIGNUM *, BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+int BN_nnmod(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+int BN_mod_add(BIGNUM *, const BIGNUM *, const BIGNUM *, const BIGNUM *,
+               BN_CTX *);
+int BN_mod_sub(BIGNUM *, const BIGNUM *, const BIGNUM *, const BIGNUM *,
+               BN_CTX *);
+int BN_mod_mul(BIGNUM *, const BIGNUM *, const BIGNUM *, const BIGNUM *,
+               BN_CTX *);
+int BN_mod_sqr(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+int BN_exp(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+int BN_mod_exp(BIGNUM *, const BIGNUM *, const BIGNUM *, const BIGNUM *,
+               BN_CTX *);
+int BN_gcd(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+BIGNUM *BN_mod_inverse(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+
+int BN_set_bit(BIGNUM *, int);
+int BN_clear_bit(BIGNUM *, int);
+
+int BN_is_bit_set(const BIGNUM *, int);
+
+int BN_mask_bits(BIGNUM *, int);
+"""
+
+MACROS = """
+int BN_num_bytes(const BIGNUM *);
+
+int BN_zero(BIGNUM *);
+int BN_one(BIGNUM *);
+int BN_mod(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+
+int BN_lshift(BIGNUM *, const BIGNUM *, int);
+int BN_lshift1(BIGNUM *, BIGNUM *);
+
+int BN_rshift(BIGNUM *, BIGNUM *, int);
+int BN_rshift1(BIGNUM *, BIGNUM *);
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/bio.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/bio.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/bio.py
@@ -0,0 +1,147 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <openssl/bio.h>
+"""
+
+TYPES = """
+typedef struct bio_st BIO;
+typedef void bio_info_cb(BIO *, int, const char *, int, long, long);
+typedef ... bio_st;
+typedef ... BIO_METHOD;
+typedef ... BUF_MEM;
+
+static const int BIO_TYPE_MEM;
+static const int BIO_TYPE_FILE;
+static const int BIO_TYPE_FD;
+static const int BIO_TYPE_SOCKET;
+static const int BIO_TYPE_CONNECT;
+static const int BIO_TYPE_ACCEPT;
+static const int BIO_TYPE_NULL;
+static const int BIO_CLOSE;
+static const int BIO_NOCLOSE;
+static const int BIO_TYPE_SOURCE_SINK;
+static const int BIO_CTRL_RESET;
+static const int BIO_CTRL_EOF;
+static const int BIO_CTRL_SET;
+static const int BIO_CTRL_SET_CLOSE;
+static const int BIO_CTRL_FLUSH;
+static const int BIO_CTRL_DUP;
+static const int BIO_CTRL_GET_CLOSE;
+static const int BIO_CTRL_INFO;
+static const int BIO_CTRL_GET;
+static const int BIO_CTRL_PENDING;
+static const int BIO_CTRL_WPENDING;
+static const int BIO_C_FILE_SEEK;
+static const int BIO_C_FILE_TELL;
+static const int BIO_TYPE_NONE;
+static const int BIO_TYPE_NBIO_TEST;
+static const int BIO_TYPE_BIO;
+static const int BIO_TYPE_DESCRIPTOR;
+static const int BIO_FLAGS_READ;
+static const int BIO_FLAGS_WRITE;
+static const int BIO_FLAGS_IO_SPECIAL;
+static const int BIO_FLAGS_RWS;
+static const int BIO_FLAGS_SHOULD_RETRY;
+static const int BIO_TYPE_NULL_FILTER;
+static const int BIO_TYPE_SSL;
+static const int BIO_TYPE_MD;
+static const int BIO_TYPE_BUFFER;
+static const int BIO_TYPE_CIPHER;
+static const int BIO_TYPE_BASE64;
+static const int BIO_TYPE_FILTER;
+"""
+
+FUNCTIONS = """
+int BIO_free(BIO *);
+void BIO_vfree(BIO *);
+void BIO_free_all(BIO *);
+BIO *BIO_push(BIO *, BIO *);
+BIO *BIO_pop(BIO *);
+BIO *BIO_next(BIO *);
+BIO *BIO_find_type(BIO *, int);
+BIO *BIO_new_file(const char *, const char *);
+BIO *BIO_new_fp(FILE *, int);
+BIO *BIO_new_fd(int, int);
+BIO *BIO_new_socket(int, int);
+long BIO_ctrl(BIO *, int, long, void *);
+long BIO_callback_ctrl(
+    BIO *,
+    int,
+    void (*)(struct bio_st *, int, const char *, int, long, long)
+);
+long BIO_int_ctrl(BIO *, int, long, int);
+size_t BIO_ctrl_pending(BIO *);
+size_t BIO_ctrl_wpending(BIO *);
+int BIO_read(BIO *, void *, int);
+int BIO_gets(BIO *, char *, int);
+int BIO_write(BIO *, const void *, int);
+int BIO_puts(BIO *, const char *);
+int BIO_method_type(const BIO *);
+"""
+
+MACROS = """
+/* Added in 1.1.0 */
+int BIO_up_ref(BIO *);
+
+/* These added const to BIO_METHOD in 1.1.0 */
+BIO *BIO_new(BIO_METHOD *);
+BIO_METHOD *BIO_s_mem(void);
+BIO_METHOD *BIO_s_file(void);
+BIO_METHOD *BIO_s_fd(void);
+BIO_METHOD *BIO_s_socket(void);
+BIO_METHOD *BIO_s_null(void);
+BIO_METHOD *BIO_f_null(void);
+BIO_METHOD *BIO_f_buffer(void);
+/* BIO_new_mem_buf became const void * in 1.0.2g */
+BIO *BIO_new_mem_buf(void *, int);
+long BIO_set_fd(BIO *, long, int);
+long BIO_get_fd(BIO *, char *);
+long BIO_set_mem_eof_return(BIO *, int);
+long BIO_get_mem_data(BIO *, char **);
+long BIO_set_mem_buf(BIO *, BUF_MEM *, int);
+long BIO_get_mem_ptr(BIO *, BUF_MEM **);
+long BIO_set_fp(BIO *, FILE *, int);
+long BIO_get_fp(BIO *, FILE **);
+long BIO_read_filename(BIO *, char *);
+long BIO_write_filename(BIO *, char *);
+long BIO_append_filename(BIO *, char *);
+long BIO_rw_filename(BIO *, char *);
+int BIO_should_read(BIO *);
+int BIO_should_write(BIO *);
+int BIO_should_io_special(BIO *);
+int BIO_retry_type(BIO *);
+int BIO_should_retry(BIO *);
+int BIO_reset(BIO *);
+int BIO_seek(BIO *, int);
+int BIO_tell(BIO *);
+int BIO_flush(BIO *);
+int BIO_eof(BIO *);
+int BIO_set_close(BIO *,long);
+int BIO_get_close(BIO *);
+int BIO_pending(BIO *);
+int BIO_wpending(BIO *);
+int BIO_get_info_callback(BIO *, bio_info_cb **);
+int BIO_set_info_callback(BIO *, bio_info_cb *);
+long BIO_get_buffer_num_lines(BIO *);
+long BIO_set_read_buffer_size(BIO *, long);
+long BIO_set_write_buffer_size(BIO *, long);
+long BIO_set_buffer_size(BIO *, long);
+long BIO_set_buffer_read_data(BIO *, void *, long);
+long BIO_set_nbio(BIO *, long);
+void BIO_set_retry_read(BIO *);
+void BIO_clear_retry_flags(BIO *);
+"""
+
+CUSTOMIZATIONS = """
+#if CRYPTOGRAPHY_OPENSSL_LESS_THAN_110PRE4 || defined(LIBRESSL_VERSION_NUMBER)
+int BIO_up_ref(BIO *b) {
+    CRYPTO_add(&b->references, 1, CRYPTO_LOCK_BIO);
+    return 1;
+}
+#endif
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/callbacks.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/callbacks.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/callbacks.py
@@ -0,0 +1,161 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+import sys
+
+import cffi
+
+INCLUDES = """
+#include <openssl/ssl.h>
+#include <openssl/x509.h>
+#include <openssl/x509_vfy.h>
+#include <openssl/crypto.h>
+"""
+
+TYPES = """
+static const long Cryptography_STATIC_CALLBACKS;
+
+/* crypto.h
+ * CRYPTO_set_locking_callback
+ * void (*cb)(int mode, int type, const char *file, int line)
+ */
+extern "Python" void Cryptography_locking_cb(int, int, const char *, int);
+
+/* pem.h
+ * int pem_password_cb(char *buf, int size, int rwflag, void *userdata);
+ */
+extern "Python" int Cryptography_pem_password_cb(char *, int, int, void *);
+
+/* rand.h
+ * int (*bytes)(unsigned char *buf, int num);
+ * int (*status)(void);
+ */
+extern "Python" int Cryptography_rand_bytes(unsigned char *, int);
+extern "Python" int Cryptography_rand_status(void);
+"""
+
+FUNCTIONS = """
+int _setup_ssl_threads(void);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+static const long Cryptography_STATIC_CALLBACKS = 1;
+"""
+
+if cffi.__version_info__ < (1, 4, 0) or sys.version_info >= (3, 5):
+    # backwards compatibility for old cffi version on PyPy
+    # and Python >=3.5 (https://github.com/pyca/cryptography/issues/2970)
+    TYPES = "static const long Cryptography_STATIC_CALLBACKS;"
+    CUSTOMIZATIONS = """static const long Cryptography_STATIC_CALLBACKS = 0;
+"""
+
+CUSTOMIZATIONS += """
+/* This code is derived from the locking code found in the Python _ssl module's
+   locking callback for OpenSSL.
+
+   Copyright 2001-2016 Python Software Foundation; All Rights Reserved.
+*/
+
+#ifdef _WIN32
+#ifdef _MSC_VER
+#ifdef inline
+#undef inline
+#endif
+#define inline __inline
+#endif
+#include <Windows.h>
+typedef CRITICAL_SECTION mutex1_t;
+static inline void mutex1_init(mutex1_t *mutex) {
+    InitializeCriticalSection(mutex);
+}
+static inline void mutex1_lock(mutex1_t *mutex) {
+    EnterCriticalSection(mutex);
+}
+static inline void mutex1_unlock(mutex1_t *mutex) {
+    LeaveCriticalSection(mutex);
+}
+#else
+#include <stdio.h>
+#include <stdlib.h>
+#include <pthread.h>
+typedef pthread_mutex_t mutex1_t;
+#define ASSERT_STATUS(call)                             \
+    if (call != 0) {                                    \
+        perror("Fatal error in _cffi_ssl: " #call);     \
+        abort();                                        \
+    }
+static inline void mutex1_init(mutex1_t *mutex) {
+#if !defined(pthread_mutexattr_default)
+#  define pthread_mutexattr_default ((pthread_mutexattr_t *)NULL)
+#endif
+    ASSERT_STATUS(pthread_mutex_init(mutex, pthread_mutexattr_default));
+}
+static inline void mutex1_lock(mutex1_t *mutex) {
+    ASSERT_STATUS(pthread_mutex_lock(mutex));
+}
+static inline void mutex1_unlock(mutex1_t *mutex) {
+    ASSERT_STATUS(pthread_mutex_unlock(mutex));
+}
+#endif
+
+static unsigned int _ssl_locks_count = 0;
+static mutex1_t *_ssl_locks = NULL;
+
+static void _ssl_thread_locking_function(int mode, int n, const char *file,
+                                         int line) {
+    /* this function is needed to perform locking on shared data
+       structures. (Note that OpenSSL uses a number of global data

From pypy.commits at gmail.com  Sat May 11 19:11:50 2019
From: pypy.commits at gmail.com (mattip)
Date: Sat, 11 May 2019 16:11:50 -0700 (PDT)
Subject: [pypy-commit] pypy cffi-libs: more hacking - should upstream
 SSL_OP_NO_SSLv2
Message-ID: <5cd756b6.1c69fb81.f773c.215a@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cffi-libs
Changeset: r96603:a5de172c4337
Date: 2019-05-10 08:08 -0700
http://bitbucket.org/pypy/pypy/changeset/a5de172c4337/

Log:	more hacking - should upstream SSL_OP_NO_SSLv2

diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
@@ -71,6 +71,7 @@
 static const long SSL_OP_MICROSOFT_SESS_ID_BUG;
 static const long SSL_OP_NETSCAPE_CHALLENGE_BUG;
 static const long SSL_OP_NETSCAPE_REUSE_CIPHER_CHANGE_BUG;
+static const long SSL_OP_NO_SSLv2;
 static const long SSL_OP_SSLREF2_REUSE_CERT_TYPE_BUG;
 static const long SSL_OP_MICROSOFT_BIG_SSLV3_BUFFER;
 static const long SSL_OP_MSIE_SSLV2_RSA_PADDING;
diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -68,7 +68,7 @@
             globals()[name[4:]] = getattr(lib, name)
 
 OP_ALL = lib.SSL_OP_ALL & ~lib.SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS
-OP_NO_SSL_v2 = lib.SSL_OP_NO_SSL_v2
+OP_NO_SSLv2 = lib.SSL_OP_NO_SSLv2
 
 SSL_CLIENT = 0
 SSL_SERVER = 1
@@ -77,7 +77,8 @@
 
 if lib.Cryptography_HAS_SSL2:
     PROTOCOL_SSLv2  = 0
-PROTOCOL_SSLv3  = 1
+if lib.Cryptography_HAS_SSL3_METHOD:
+    PROTOCOL_SSLv3  = 1
 PROTOCOL_SSLv23 = 2
 PROTOCOL_TLS    = PROTOCOL_SSLv23
 PROTOCOL_TLSv1    = 3
@@ -156,7 +157,7 @@
     ffi.memmove(buf, password, len(password))
     return len(password)
 
-if lib.Cryptography_STATIC_CALLBACKS:
+if 0: # lib.Cryptography_STATIC_CALLBACKS:
     ffi.def_extern(_Cryptography_pem_password_cb)
     Cryptography_pem_password_cb = lib.Cryptography_pem_password_cb
 else:
@@ -530,7 +531,7 @@
         short_name = lib.SSL_COMP_get_name(comp_method)
         if short_name == ffi.NULL:
             return None
-        return _cstr_decode_fs(short_name)
+        return _str_from_buf(short_name)
 
     def version(self):
         if self.ssl == ffi.NULL:
@@ -781,7 +782,7 @@
             method = lib.TLSv1_1_method()
         elif lib.Cryptography_HAS_TLSv1_2 and protocol == PROTOCOL_TLSv1_2 :
             method = lib.TLSv1_2_method()
-        elif protocol == PROTOCOL_SSLv3 and lib.Cryptography_HAS_SSL3_METHOD:
+        elif lib.Cryptography_HAS_SSL3_METHOD and protocol == PROTOCOL_SSLv3:
             method = lib.SSLv3_method()
         elif lib.Cryptography_HAS_SSL2 and protocol == PROTOCOL_SSLv2:
             method = lib.SSLv2_method()
@@ -812,7 +813,7 @@
         options = lib.SSL_OP_ALL & ~lib.SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS
         if not lib.Cryptography_HAS_SSL2 or protocol != PROTOCOL_SSLv2:
             options |= lib.SSL_OP_NO_SSLv2
-        if protocol != PROTOCOL_SSLv3:
+        if not lib.Cryptography_HAS_SSL3_METHOD or protocol != PROTOCOL_SSLv3:
             options |= lib.SSL_OP_NO_SSLv3
         # Minimal security flags for server and client side context.
         # Client sockets ignore server-side parameters.
@@ -976,7 +977,7 @@
                 _errno = ffi.errno
                 if _errno:
                     lib.ERR_clear_error()
-                    raise OSError(_errno, "Error")
+                    raise IOError(_errno, "Error")
                 else:
                     raise ssl_error(None)
 
@@ -991,7 +992,7 @@
                 _errno = ffi.errno
                 if _errno:
                     lib.ERR_clear_error()
-                    raise OSError(_errno, None)
+                    raise IOError(_errno, None)
                 else:
                     raise ssl_error(None)
 
@@ -1016,7 +1017,7 @@
         if cadata is None:
             ca_file_type = -1
         else:
-            if not isinstance(cadata, str):
+            if not isinstance(cadata, unicode):
                 ca_file_type = lib.SSL_FILETYPE_ASN1
             else:
                 ca_file_type = lib.SSL_FILETYPE_PEM
@@ -1024,8 +1025,11 @@
                     cadata = cadata.encode('ascii')
                 except UnicodeEncodeError:
                     raise TypeError("cadata should be a ASCII string or a bytes-like object")
-        if cafile is None and capath is None and cadata is None:
-            raise TypeError("cafile and capath cannot be both omitted")
+        if cafile is None and capath is None:
+            if cadata is None:
+                raise TypeError("cafile and capath cannot be both omitted")
+            if not cadata:
+                raise ssl_error(None)
         # load from cadata
         if cadata is not None:
             buf = _str_to_ffi_buffer(cadata)
@@ -1046,7 +1050,7 @@
                 _errno = ffi.errno
                 if _errno:
                     lib.ERR_clear_error()
-                    raise OSError(_errno, '')
+                    raise IOError(_errno, '')
                 else:
                     raise ssl_error(None)
 
@@ -1143,7 +1147,7 @@
         if bio == ffi.NULL:
             _errno = ffi.errno
             lib.ERR_clear_error()
-            raise OSError(_errno, '')
+            raise IOError(_errno, '')
         try:
             dh = lib.PEM_read_bio_DHparams(bio, ffi.NULL, ffi.NULL, ffi.NULL)
         finally:
@@ -1473,16 +1477,16 @@
 
 def get_default_verify_paths():
 
-    ofile_env = _cstr_decode_fs(lib.X509_get_default_cert_file_env())
+    ofile_env = _str_from_buf(lib.X509_get_default_cert_file_env())
     if ofile_env is None:
         return None
-    ofile = _cstr_decode_fs(lib.X509_get_default_cert_file())
+    ofile = _str_from_buf(lib.X509_get_default_cert_file())
     if ofile is None:
         return None
-    odir_env = _cstr_decode_fs(lib.X509_get_default_cert_dir_env())
+    odir_env = _str_from_buf(lib.X509_get_default_cert_dir_env())
     if odir_env is None:
         return None
-    odir = _cstr_decode_fs(lib.X509_get_default_cert_dir())
+    odir = _str_from_buf(lib.X509_get_default_cert_dir())
     if odir is None:
         return odir
     return (ofile_env, ofile, odir_env, odir);
diff --git a/lib_pypy/_cffi_ssl/_stdssl/utility.py b/lib_pypy/_cffi_ssl/_stdssl/utility.py
--- a/lib_pypy/_cffi_ssl/_stdssl/utility.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/utility.py
@@ -8,13 +8,13 @@
     return _str_with_len(ffi.cast("char*",data), length)
 
 def _str_with_len(char_ptr, length):
-    return ffi.buffer(char_ptr, length)[:].decode('utf-8')
+    return ffi.buffer(char_ptr, length)[:]
 
 def _bytes_with_len(char_ptr, length):
     return ffi.buffer(char_ptr, length)[:]
 
 def _str_to_ffi_buffer(view):
-    if isinstance(view, str):
+    if isinstance(view, unicode):
         return ffi.from_buffer(view.encode())
     elif isinstance(view, memoryview):
         # NOTE pypy limitation StringBuffer does not allow
@@ -25,7 +25,7 @@
     return ffi.from_buffer(view)
 
 def _str_from_buf(buf):
-    return ffi.string(buf).decode('utf-8')
+    return ffi.string(buf)
 
 def _cstr_decode_fs(buf):
     if buf == ffi.NULL:
diff --git a/lib_pypy/_ssl/__init__.py b/lib_pypy/_ssl/__init__.py
--- a/lib_pypy/_ssl/__init__.py
+++ b/lib_pypy/_ssl/__init__.py
@@ -3,6 +3,8 @@
 from _cffi_ssl import _stdssl
 from _cffi_ssl._stdssl import *
 
+OP_SINGLE_DH_USE = lib.SSL_OP_SINGLE_DH_USE
+OP_SINGLE_ECDH_USE = lib.SSL_OP_SINGLE_ECDH_USE
 
 try: from __pypy__ import builtinify
 except ImportError: builtinify = lambda f: f

From pypy.commits at gmail.com  Sat May 11 19:11:52 2019
From: pypy.commits at gmail.com (mattip)
Date: Sat, 11 May 2019 16:11:52 -0700 (PDT)
Subject: [pypy-commit] pypy default: more typo
Message-ID: <5cd756b8.1c69fb81.741d2.38b8@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r96604:19da1ad99ab7
Date: 2019-05-11 16:10 -0700
http://bitbucket.org/pypy/pypy/changeset/19da1ad99ab7/

Log:	more typo

diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py
--- a/rpython/translator/c/gc.py
+++ b/rpython/translator/c/gc.py
@@ -464,7 +464,7 @@
         return '(char *)(%s) += sizeof(pypy_ss_t);' % (funcgen.gcpol_ss,)
 
     def OP_GC_LEAVE_ROOTS_FRAME(self, funcgen, op):
-        return (char *)(%s) -= sizeof(pypy_ss_t);' % (funcgen.gcpol_ss,)
+        return '(char *)(%s) -= sizeof(pypy_ss_t);' % (funcgen.gcpol_ss,)
 
     def OP_GC_SAVE_ROOT(self, funcgen, op):
         num = op.args[0].value

From pypy.commits at gmail.com  Sun May 12 21:13:47 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 12 May 2019 18:13:47 -0700 (PDT)
Subject: [pypy-commit] pypy default: actually test before committing
Message-ID: <5cd8c4cb.1c69fb81.82251.4bdb@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r96605:ad8eb33a762d
Date: 2019-05-12 18:12 -0700
http://bitbucket.org/pypy/pypy/changeset/ad8eb33a762d/

Log:	actually test before committing

diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py
--- a/rpython/translator/c/gc.py
+++ b/rpython/translator/c/gc.py
@@ -450,7 +450,7 @@
         # XXX hard-code the field name here
         gcpol_ss = '%s->gcd_inst_root_stack_top' % funcgen.expr(c_gcdata)
         #
-        yield ('typedef struct { void %s; } pypy_ss_t;'
+        yield ('typedef struct { char %s; } pypy_ss_t;'
                    % ', '.join(['*s%d' % i for i in range(numcolors)]))
         funcgen.gcpol_ss = gcpol_ss
 
@@ -461,15 +461,15 @@
         raise Exception("gc_pop_roots should be removed by postprocess_graph")
 
     def OP_GC_ENTER_ROOTS_FRAME(self, funcgen, op):
-        return '(char *)(%s) += sizeof(pypy_ss_t);' % (funcgen.gcpol_ss,)
+        return '(%s) += sizeof(pypy_ss_t);' % (funcgen.gcpol_ss,)
 
     def OP_GC_LEAVE_ROOTS_FRAME(self, funcgen, op):
-        return '(char *)(%s) -= sizeof(pypy_ss_t);' % (funcgen.gcpol_ss,)
+        return '(%s) -= sizeof(pypy_ss_t);' % (funcgen.gcpol_ss,)
 
     def OP_GC_SAVE_ROOT(self, funcgen, op):
         num = op.args[0].value
         exprvalue = funcgen.expr(op.args[1])
-        return '((pypy_ss_t *)%s)[-1].s%d = (void *)%s;' % (
+        return '((pypy_ss_t *)%s)[-1].s%d = (char *)%s;' % (
             funcgen.gcpol_ss, num, exprvalue)
 
     def OP_GC_RESTORE_ROOT(self, funcgen, op):

From pypy.commits at gmail.com  Sun May 12 22:31:42 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 12 May 2019 19:31:42 -0700 (PDT)
Subject: [pypy-commit] pypy default: add some pieces needed to start
 building opencv-python with scikit-build
Message-ID: <5cd8d70e.1c69fb81.3aec9.e281@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r96606:b746e83d4c54
Date: 2019-05-12 19:30 -0700
http://bitbucket.org/pypy/pypy/changeset/b746e83d4c54/

Log:	add some pieces needed to start building opencv-python with scikit-
	build

diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -71,7 +71,7 @@
     g['AR'] = "ar"
     g['ARFLAGS'] = "rc"
     g['EXE'] = ""
-    g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
+    g['LIBDIR'] = os.path.join(sys.prefix, 'bin') # where is the shared library
     g['VERSION'] = get_python_version()
 
     if sys.platform[:6] == "darwin":
diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py
--- a/lib-python/2.7/sysconfig.py
+++ b/lib-python/2.7/sysconfig.py
@@ -483,6 +483,7 @@
         _CONFIG_VARS['projectbase'] = _PROJECT_BASE
         _CONFIG_VARS['implementation'] = _get_implementation()
         _CONFIG_VARS['implementation_lower'] = _get_implementation().lower()
+        _CONFIG_VARS['LIBRARY'] = ''
 
         if os.name in ('nt', 'os2'):
             _init_non_posix(_CONFIG_VARS)

From pypy.commits at gmail.com  Sun May 12 23:38:29 2019
From: pypy.commits at gmail.com (whitequark)
Date: Sun, 12 May 2019 20:38:29 -0700 (PDT)
Subject: [pypy-commit] pypy fix-literal-prev_digit-underscore: Fix a crash
 on literals like 0xffff_ffff_ff20_0000.
Message-ID: <5cd8e6b5.1c69fb81.bd340.edf5@mx.google.com>

Author: whitequark <whitequark at whitequark.org>
Branch: fix-literal-prev_digit-underscore
Changeset: r96607:2c00b58bf83c
Date: 2019-05-13 03:34 +0000
http://bitbucket.org/pypy/pypy/changeset/2c00b58bf83c/

Log:	Fix a crash on literals like 0xffff_ffff_ff20_0000.

diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -126,6 +126,9 @@
         for c in expressions.constants:
             yield (self.simple_test, "x="+c, "x", eval(c))
 
+    def test_const_underscore(self):
+        yield (self.simple_test, "x=0xffff_ffff_ff20_0000", "x", 0xffffffffff200000)
+
     def test_neg_sys_maxint(self):
         import sys
         stmt = "x = %s" % (-sys.maxint-1)
diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py
--- a/rpython/rlib/rstring.py
+++ b/rpython/rlib/rstring.py
@@ -579,6 +579,11 @@
         assert i >= 0
         self.i = i
         c = self.s[i]
+        if self.allow_underscores and c == '_':
+            i = self.i - 1
+            assert i >= 0
+            self.i = i
+            c = self.s[i]
         digit = ord(c)
         if '0' <= c <= '9':
             digit -= ord('0')

From pypy.commits at gmail.com  Mon May 13 00:10:55 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 12 May 2019 21:10:55 -0700 (PDT)
Subject: [pypy-commit] pypy default: this enables building opencv2 since
 they use these macros
Message-ID: <5cd8ee4f.1c69fb81.84fa2.55ce@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r96608:3175515ea703
Date: 2019-05-12 20:19 -0700
http://bitbucket.org/pypy/pypy/changeset/3175515ea703/

Log:	this enables building opencv2 since they use these macros

diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -391,6 +391,16 @@
 PyAPI_FUNC(PyVarObject *) PyObject_InitVar(PyVarObject *,
                                            PyTypeObject *, Py_ssize_t);
 
+/*
+ * On CPython with Py_REF_DEBUG these use _PyRefTotal, _Py_NegativeRefcount,
+ * _Py_GetRefTotal, ...
+ * So far we ignore Py_REF_DEBUG
+ */
+
+#define _Py_INC_REFTOTAL
+#define _Py_DEC_REFTOTAL
+#define _Py_REF_DEBUG_COMMA
+#define _Py_CHECK_REFCNT(OP)    /* a semicolon */;
 
 /* PyPy internal ----------------------------------- */
 PyAPI_FUNC(int) PyPyType_Register(PyTypeObject *);

From pypy.commits at gmail.com  Mon May 13 17:02:29 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 13 May 2019 14:02:29 -0700 (PDT)
Subject: [pypy-commit] pypy fix-literal-prev_digit-underscore: document and
 close branch to be merged
Message-ID: <5cd9db65.1c69fb81.a6d7.6da5@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: fix-literal-prev_digit-underscore
Changeset: r96609:b41ce6edfaf2
Date: 2019-05-13 14:00 -0700
http://bitbucket.org/pypy/pypy/changeset/b41ce6edfaf2/

Log:	document and close branch to be merged

diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -2,14 +2,9 @@
 What's new in PyPy3 7.1+
 ========================
 
-.. this is the revision after release-pypy3.6-v7.1
-.. startrev: d642a3c217cb
+.. this is the revision after release-pypy3.6-v7.1.1
+.. startrev: db5a1e7fbbd0
 
-.. branch: zlib-make-py3-go-boom
+.. branch: fix-literal-prev_digit-underscore
 
-Complain if you try to copy a flushed zlib decompress on py3
-
-.. branch: winoverlapped
-
-Add support for async (overlapped) IO on Windows.
-
+Fix parsing for converting strings with underscore into ints

From pypy.commits at gmail.com  Mon May 13 17:02:31 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 13 May 2019 14:02:31 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: merge fix-literal-prev_digit-underscore
 which fixes converting strings with '_' to ints
Message-ID: <5cd9db67.1c69fb81.39bb4.1b9a@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r96610:c0cb46236cc7
Date: 2019-05-13 14:01 -0700
http://bitbucket.org/pypy/pypy/changeset/c0cb46236cc7/

Log:	merge fix-literal-prev_digit-underscore which fixes converting
	strings with '_' to ints

diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -2,14 +2,9 @@
 What's new in PyPy3 7.1+
 ========================
 
-.. this is the revision after release-pypy3.6-v7.1
-.. startrev: d642a3c217cb
+.. this is the revision after release-pypy3.6-v7.1.1
+.. startrev: db5a1e7fbbd0
 
-.. branch: zlib-make-py3-go-boom
+.. branch: fix-literal-prev_digit-underscore
 
-Complain if you try to copy a flushed zlib decompress on py3
-
-.. branch: winoverlapped
-
-Add support for async (overlapped) IO on Windows.
-
+Fix parsing for converting strings with underscore into ints
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -126,6 +126,9 @@
         for c in expressions.constants:
             yield (self.simple_test, "x="+c, "x", eval(c))
 
+    def test_const_underscore(self):
+        yield (self.simple_test, "x=0xffff_ffff_ff20_0000", "x", 0xffffffffff200000)
+
     def test_neg_sys_maxint(self):
         import sys
         stmt = "x = %s" % (-sys.maxint-1)
diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py
--- a/rpython/rlib/rstring.py
+++ b/rpython/rlib/rstring.py
@@ -579,6 +579,11 @@
         assert i >= 0
         self.i = i
         c = self.s[i]
+        if self.allow_underscores and c == '_':
+            i = self.i - 1
+            assert i >= 0
+            self.i = i
+            c = self.s[i]
         digit = ord(c)
         if '0' <= c <= '9':
             digit -= ord('0')

From pypy.commits at gmail.com  Mon May 13 21:21:10 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 13 May 2019 18:21:10 -0700 (PDT)
Subject: [pypy-commit] pypy default: document merged branch
Message-ID: <5cda1806.1c69fb81.f1c62.3efd@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r96611:b49be935188c
Date: 2019-05-13 18:09 -0700
http://bitbucket.org/pypy/pypy/changeset/b49be935188c/

Log:	document merged branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -21,3 +21,7 @@
 
 Test and reduce the probability of a deadlock when acquiring a semaphore by
 moving global state changes closer to the actual aquire.
+
+.. branch: shadowstack-issue2722
+
+Make the shadowstack size more dynamic

From pypy.commits at gmail.com  Mon May 13 21:21:12 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 13 May 2019 18:21:12 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: merge default into py3.6
Message-ID: <5cda1808.1c69fb81.b1f47.3e9d@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r96612:646010cf591d
Date: 2019-05-13 18:17 -0700
http://bitbucket.org/pypy/pypy/changeset/646010cf591d/

Log:	merge default into py3.6

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -25,3 +25,7 @@
 
 Test and reduce the probability of a deadlock when acquiring a semaphore by
 moving global state changes closer to the actual aquire.
+
+.. branch: shadowstack-issue2722
+
+Make the shadowstack size more dynamic
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -384,6 +384,16 @@
 PyAPI_FUNC(int) PyObject_CallFinalizerFromDealloc(PyObject *);
 #endif
 
+/*
+ * On CPython with Py_REF_DEBUG these use _PyRefTotal, _Py_NegativeRefcount,
+ * _Py_GetRefTotal, ...
+ * So far we ignore Py_REF_DEBUG
+ */
+
+#define _Py_INC_REFTOTAL
+#define _Py_DEC_REFTOTAL
+#define _Py_REF_DEBUG_COMMA
+#define _Py_CHECK_REFCNT(OP)    /* a semicolon */;
 
 /* PyPy internal ----------------------------------- */
 PyAPI_FUNC(int) PyPyType_Register(PyTypeObject *);
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -55,9 +55,15 @@
 reserves 768KB of stack space, which should suffice (on Linux,
 depending on the compiler settings) for ~1400 calls.  Setting the
 value to N reserves N/1000 times 768KB of stack space.
+
+Note that there are other factors that also limit the stack size.
+The operating system typically sets a maximum which can be changed
+manually (e.g. with "ulimit" on Linux) for the main thread.  For other
+threads you can configure the limit by calling "threading.stack_size()".
 """
     from rpython.rlib.rstack import _stack_set_length_fraction
     from rpython.rlib.rstackovf import StackOverflow
+    from rpython.rlib.rgc import increase_root_stack_depth
     if new_limit <= 0:
         raise oefmt(space.w_ValueError, "recursion limit must be positive")
     try:
@@ -69,6 +75,7 @@
         raise oefmt(space.w_RecursionError,
                     "maximum recursion depth exceeded")
     space.sys.recursionlimit = new_limit
+    increase_root_stack_depth(int(new_limit * 0.001 * 163840))
 
 def getrecursionlimit(space):
     """Return the last value set by setrecursionlimit().
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -572,6 +572,8 @@
             self.move_out_of_nursery_ptr = getfn(GCClass.move_out_of_nursery,
                                               [s_gc, SomeAddress()],
                                               SomeAddress())
+        if hasattr(self.root_walker, 'build_increase_root_stack_depth_ptr'):
+            self.root_walker.build_increase_root_stack_depth_ptr(getfn)
 
 
     def create_custom_trace_funcs(self, gc, rtyper):
@@ -1652,6 +1654,12 @@
         else:
             hop.rename("same_as")
 
+    def gct_gc_increase_root_stack_depth(self, hop):
+        if not hasattr(self.root_walker, 'gc_increase_root_stack_depth_ptr'):
+            return
+        hop.genop("direct_call",
+                  [self.root_walker.gc_increase_root_stack_depth_ptr,
+                   hop.spaceop.args[0]])
 
 
 class TransformerLayoutBuilder(gctypelayout.TypeLayoutBuilder):
diff --git a/rpython/memory/gctransform/shadowstack.py b/rpython/memory/gctransform/shadowstack.py
--- a/rpython/memory/gctransform/shadowstack.py
+++ b/rpython/memory/gctransform/shadowstack.py
@@ -140,6 +140,7 @@
         # this is a dict {tid: SHADOWSTACKREF}, where the tid for the
         # current thread may be missing so far
         gcdata.thread_stacks = None
+        shadow_stack_pool.has_threads = True
 
         # Return the thread identifier, as an integer.
         get_tid = rthread.get_ident
@@ -252,6 +253,15 @@
         self.gc_modified_shadowstack_ptr = getfn(gc_modified_shadowstack,
                                                  [], annmodel.s_None)
 
+    def build_increase_root_stack_depth_ptr(self, getfn):
+        shadow_stack_pool = self.shadow_stack_pool
+        def gc_increase_root_stack_depth(new_size):
+            shadow_stack_pool.increase_root_stack_depth(new_size)
+
+        self.gc_increase_root_stack_depth_ptr = getfn(
+                gc_increase_root_stack_depth, [annmodel.SomeInteger()],
+                annmodel.s_None)
+
     def postprocess_graph(self, gct, graph, any_inlining):
         from rpython.memory.gctransform import shadowcolor
         if any_inlining:
@@ -269,6 +279,7 @@
     """
     _alloc_flavor_ = "raw"
     root_stack_depth = 163840
+    has_threads = False
 
     def __init__(self, gcdata):
         self.unused_full_stack = llmemory.NULL
@@ -337,6 +348,44 @@
             if self.unused_full_stack == llmemory.NULL:
                 raise MemoryError
 
+    def increase_root_stack_depth(self, new_depth):
+        if new_depth <= self.root_stack_depth:
+            return     # can't easily decrease the size
+        if self.unused_full_stack:
+            llmemory.raw_free(self.unused_full_stack)
+            self.unused_full_stack = llmemory.NULL
+        used = self.gcdata.root_stack_top - self.gcdata.root_stack_base
+        addr = self._resize(self.gcdata.root_stack_base, used, new_depth)
+        self.gcdata.root_stack_base = addr
+        self.gcdata.root_stack_top  = addr + used
+        # no gc operations above: we just switched shadowstacks
+        if self.has_threads:
+            self._resize_thread_shadowstacks(new_depth)
+        self.root_stack_depth = new_depth
+
+    def _resize_thread_shadowstacks(self, new_depth):
+        if self.gcdata.thread_stacks is not None:
+            for ssref in self.gcdata.thread_stacks.values():
+                if ssref.base:
+                    used = ssref.top - ssref.base
+                    addr = self._resize(ssref.base, used, new_depth)
+                    ssref.base = addr
+                    ssref.top = addr + used
+    _resize_thread_shadowstacks._dont_inline_ = True
+
+    def _resize(self, base, used, new_depth):
+        new_size = sizeofaddr * new_depth
+        ll_assert(used <= new_size, "shadowstack resize: overflow detected")
+        addr = llmemory.raw_malloc(new_size)
+        if addr == llmemory.NULL:
+            raise MemoryError
+        # note that we don't know the total memory size of 'base', but we
+        # know the size of the part that is used right now, and we only need
+        # to copy that
+        llmemory.raw_memmove(base, addr, used)
+        llmemory.raw_free(base)
+        return addr
+
 
 def get_shadowstackref(root_walker, gctransformer):
     if hasattr(gctransformer, '_SHADOWSTACKREF'):
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -690,6 +690,13 @@
         hop.exception_cannot_occur()
         return hop.genop('gc_move_out_of_nursery', hop.args_v, resulttype=hop.r_result)
 
+ at jit.dont_look_inside
+def increase_root_stack_depth(new_depth):
+    """Shadowstack: make sure the size of the shadowstack is at least
+    'new_depth' pointers."""
+    from rpython.rtyper.lltypesystem.lloperation import llop
+    llop.gc_increase_root_stack_depth(lltype.Void, new_depth)
+
 # ____________________________________________________________
 
 
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -1226,6 +1226,9 @@
     def op_gc_move_out_of_nursery(self, obj):
         raise NotImplementedError("gc_move_out_of_nursery")
 
+    def op_gc_increase_root_stack_depth(self, new_depth):
+        raise NotImplementedError("gc_increase_root_stack_depth")
+
     def op_revdb_stop_point(self, *args):
         pass
     def op_revdb_send_answer(self, *args):
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -530,6 +530,7 @@
     'gc_rawrefcount_next_dead':         LLOp(),
 
     'gc_move_out_of_nursery':           LLOp(),
+    'gc_increase_root_stack_depth':     LLOp(canrun=True),
 
     'gc_push_roots'        : LLOp(),  # temporary: list of roots to save
     'gc_pop_roots'         : LLOp(),  # temporary: list of roots to restore
diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py
--- a/rpython/rtyper/lltypesystem/opimpl.py
+++ b/rpython/rtyper/lltypesystem/opimpl.py
@@ -776,6 +776,9 @@
 def op_gc_move_out_of_nursery(obj):
     return obj
 
+def op_gc_increase_root_stack_depth(new_depth):
+    pass
+
 def op_revdb_do_next_call():
     pass
 
diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py
--- a/rpython/translator/c/gc.py
+++ b/rpython/translator/c/gc.py
@@ -450,9 +450,8 @@
         # XXX hard-code the field name here
         gcpol_ss = '%s->gcd_inst_root_stack_top' % funcgen.expr(c_gcdata)
         #
-        yield ('typedef struct { void %s; } pypy_ss_t;'
+        yield ('typedef struct { char %s; } pypy_ss_t;'
                    % ', '.join(['*s%d' % i for i in range(numcolors)]))
-        yield 'pypy_ss_t *ss;'
         funcgen.gcpol_ss = gcpol_ss
 
     def OP_GC_PUSH_ROOTS(self, funcgen, op):
@@ -462,26 +461,27 @@
         raise Exception("gc_pop_roots should be removed by postprocess_graph")
 
     def OP_GC_ENTER_ROOTS_FRAME(self, funcgen, op):
-        return 'ss = (pypy_ss_t *)%s; %s = (void *)(ss+1);' % (
-            funcgen.gcpol_ss, funcgen.gcpol_ss)
+        return '(%s) += sizeof(pypy_ss_t);' % (funcgen.gcpol_ss,)
 
     def OP_GC_LEAVE_ROOTS_FRAME(self, funcgen, op):
-        return '%s = (void *)ss;' % funcgen.gcpol_ss
+        return '(%s) -= sizeof(pypy_ss_t);' % (funcgen.gcpol_ss,)
 
     def OP_GC_SAVE_ROOT(self, funcgen, op):
         num = op.args[0].value
         exprvalue = funcgen.expr(op.args[1])
-        return 'ss->s%d = (void *)%s;\t/* gc_save_root */' % (num, exprvalue)
+        return '((pypy_ss_t *)%s)[-1].s%d = (char *)%s;' % (
+            funcgen.gcpol_ss, num, exprvalue)
 
     def OP_GC_RESTORE_ROOT(self, funcgen, op):
         num = op.args[0].value
         exprvalue = funcgen.expr(op.args[1])
         typename = funcgen.db.gettype(op.args[1].concretetype)
-        result = '%s = (%s)ss->s%d;' % (exprvalue, cdecl(typename, ''), num)
+        result = '%s = (%s)((pypy_ss_t *)%s)[-1].s%d;' % (
+            exprvalue, cdecl(typename, ''), funcgen.gcpol_ss, num)
         if isinstance(op.args[1], Constant):
-            return '/* %s\t* gc_restore_root */' % result
+            return '/* %s */' % result
         else:
-            return '%s\t/* gc_restore_root */' % result
+            return result
 
 
 class AsmGcRootFrameworkGcPolicy(BasicFrameworkGcPolicy):
diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -1912,6 +1912,45 @@
     def test_total_gc_time(self):
         res = self.run("total_gc_time")
         assert res > 0 # should take a few microseconds
+
+    def define_increase_root_stack_depth(cls):
+        class X:
+            pass
+        def g(n):
+            if n <= 0:
+                return None
+            x = X()
+            x.n = n
+            x.next = g(n - 1)
+            return x
+        def f(depth):
+            from rpython.rlib.rstack import _stack_set_length_fraction
+            _stack_set_length_fraction(50.0)
+            # ^^^ the default is enough for at least 10'000 (but less than
+            # 100'000) recursions of the simple function g().  We multiply
+            # it by 50.0 to make sure that 200'000 works.  The default
+            # shadowstack depth is 163'840 entries, so 200'000 overflows
+            # that default shadowstack depth, and gives a segfault unless
+            # the following line works too.
+            from rpython.rlib.rgc import increase_root_stack_depth
+            increase_root_stack_depth(depth + 100)
+            #
+            g(depth)
+            return 42
+        return f
+
+    def test_increase_root_stack_depth(self):
+        if not sys.platform.startswith('linux'):
+            py.test.skip("linux only")
+        #
+        def myrunner(args):
+            args1 = ['/bin/bash', '-c', 'ulimit -s unlimited && %s' %
+                     (' '.join(args),)]
+            return subprocess.check_output(args1)
+        res = self.run("increase_root_stack_depth", 200000, runner=myrunner)
+        assert res == 42
+
+
 # ____________________________________________________________________
 
 class TaggedPointersTest(object):

From pypy.commits at gmail.com  Tue May 14 11:36:34 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 14 May 2019 08:36:34 -0700 (PDT)
Subject: [pypy-commit] pypy default: document the issues around
 major.minor.micro release version numbers
Message-ID: <5cdae082.1c69fb81.e657e.7327@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r96614:2680d2fce15d
Date: 2019-05-14 08:35 -0700
http://bitbucket.org/pypy/pypy/changeset/2680d2fce15d/

Log:	document the issues around major.minor.micro release version numbers

diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst
--- a/pypy/doc/how-to-release.rst
+++ b/pypy/doc/how-to-release.rst
@@ -8,6 +8,12 @@
 a branch named like release-pypy3.5-v2.x or release-pypy3.5-v4.x, and each
 release is tagged, for instance release-pypy3.5-v4.0.1. 
 
+The release version number should be bumped. A micro release increment means
+there were no changes that justify rebuilding c-extension wheels, since
+the wheels are marked with only major.minor version numbers. It is ofen not
+clear what constitues a "major" release verses a "minor" release, the release
+manager can make that call.
+
 After release, inevitably there are bug fixes. It is the responsibility of
 the commiter who fixes a bug to make sure this fix is on the release branch,
 so that we can then create a tagged bug-fix release, which will hopefully
@@ -78,6 +84,8 @@
 
 * Maybe bump the SOABI number in module/imp/importing. This has many
   implications, so make sure the PyPy community agrees to the change.
+  Wheels will use the major.minor release numbers in the name, so bump
+  them if there is an incompatible change to cpyext.
 
 * Update and write documentation
 

From pypy.commits at gmail.com  Wed May 15 05:48:42 2019
From: pypy.commits at gmail.com (fijal)
Date: Wed, 15 May 2019 02:48:42 -0700 (PDT)
Subject: [pypy-commit] pypy arm64: array basics
Message-ID: <5cdbe07a.1c69fb81.e8e3d.9a38@mx.google.com>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: arm64
Changeset: r96615:411dd68d082c
Date: 2019-05-15 09:47 +0000
http://bitbucket.org/pypy/pypy/changeset/411dd68d082c/

Log:	array basics

diff --git a/rpython/jit/backend/aarch64/TODO b/rpython/jit/backend/aarch64/TODO
--- a/rpython/jit/backend/aarch64/TODO
+++ b/rpython/jit/backend/aarch64/TODO
@@ -1,3 +1,3 @@
 * int_add - IMM
 * int_cmp - IMM
-* *_ovf - merging operations
\ No newline at end of file
+* guard_nonnull_class - think about a better way
\ No newline at end of file
diff --git a/rpython/jit/backend/aarch64/codebuilder.py b/rpython/jit/backend/aarch64/codebuilder.py
--- a/rpython/jit/backend/aarch64/codebuilder.py
+++ b/rpython/jit/backend/aarch64/codebuilder.py
@@ -7,8 +7,6 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.tool.udir import udir
 
-PC_OFFSET = 0 # XXX
-
 class AbstractAarch64Builder(object):
     def write32(self, word):
         self.writechar(chr(word & 0xFF))
@@ -43,7 +41,7 @@
     def STR_size_rr(self, scale, rt, rn, rm):
         base = 0b111000001
         assert 0 <= scale <= 3
-        self.wirte32((scale << 30) | (base << 21) | (rm << 16) | (0b11 << 13) |
+        self.write32((scale << 30) | (base << 21) | (rm << 16) | (0b11 << 13) |
                      (0b010 << 10) | (rn << 5) | rt)
 
     def STR_size_ri(self, scale, rt, rn, imm):
@@ -112,18 +110,37 @@
         assert immed & 0x7 == 0
         self.write32((base << 22) | (immed >> 3 << 10) | (rn << 5) | rt)
 
+    def LDRB_ri(self, rt, rn, immed):
+        base = 0b0011100101
+        assert 0 <= immed <= 1<<12
+        self.write32((base << 22) | (immed << 10) | (rn << 5) | rt)
+
+    def LDRSH_ri(self, rt, rn, immed):
+        base = 0b0111100110
+        assert 0 <= immed <= 1<<13
+        assert immed & 0b1 == 0
+        self.write32((base << 22) | (immed >> 1 << 10) | (rn << 5) | rt)
+
     def LDR_rr(self, rt, rn, rm):
-        xxx
+        base = 0b11111000011
+        self.write32((base << 21) | (rm << 16) | (0b011010 << 10) | (rn << 5) | rt)
 
-    def LDR_size_ri(self, size, rt, rn, ofs):
-        assert 0 <= size <= 3
-        assert 0 <= ofs <= 4096
-        base = 0b11100101
-        self.write32((size << 30) | (base << 22) | (ofs >> size << 10) | (rn << 5) | rt)
+    def LDRB_rr(self, rt, rn, rm):
+        base = 0b00111000011
+        self.write32((base << 21) | (rm << 16) | (0b011010 << 10) | (rn << 5) | rt)
 
-    def LDR_size_rr(self, size, rt, rn, rm):
-        xxx
-    
+    def LDRSW_rr(self, rt, rn, rm):
+        base = 0b10111000101
+        self.write32((base << 21) | (rm << 16) | (0b011010 << 10) | (rn << 5) | rt)
+
+    def LDRSH_rr(self, rt, rn, rm):
+        base = 0b01111000101
+        self.write32((base << 21) | (rm << 16) | (0b011010 << 10) | (rn << 5) | rt)
+
+    def LDRSB_rr(self, rt, rn, rm):
+        base = 0b00111000101
+        self.write32((base << 21) | (rm << 16) | (0b011010 << 10) | (rn << 5) | rt)
+
     def LDR_r_literal(self, rt, offset):
         base = 0b01011000
         assert -(1 << 20) <= offset < (1<< 20)
@@ -237,14 +254,12 @@
     def B_ofs(self, ofs):
         base = 0b000101
         assert ofs & 0x3 == 0
-        pos = self.currpos()
-        target_ofs = ofs - (pos + PC_OFFSET)
-        assert -(1 << (26 + 2)) < target_ofs < 1<<(26 + 2)
-        if target_ofs < 0:
-            target_ofs = (1 << 26) - (-target_ofs >> 2)
+        assert -(1 << (26 + 2)) < ofs < 1<<(26 + 2)
+        if ofs < 0:
+            ofs = (1 << 26) - (-ofs >> 2)
         else:
-            target_ofs = target_ofs >> 2
-        self.write32((base << 26) | target_ofs)
+            ofs = ofs >> 2
+        self.write32((base << 26) | ofs)
 
     def B_ofs_cond(self, ofs, cond):
         base = 0b01010100
@@ -255,6 +270,11 @@
             xxx
         self.write32((base << 24) | (imm << 5) | cond)
 
+    def B(self, target):
+        target = rffi.cast(lltype.Signed, target)
+        self.gen_load_int_full(r.ip0.value, target)
+        self.BR_r(r.ip0.value)        
+
     def BL(self, target):
         # XXX use the IMM version if close enough
         target = rffi.cast(lltype.Signed, target)
@@ -265,7 +285,7 @@
         base = 0b1101011000111111000000
         self.write32((base << 10) | (reg << 5))
 
-    def BR(self, reg):
+    def BR_r(self, reg):
         base = 0b1101011000011111000000
         self.write32((base << 10) | (reg << 5))
 
diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py
--- a/rpython/jit/backend/aarch64/opassembler.py
+++ b/rpython/jit/backend/aarch64/opassembler.py
@@ -127,6 +127,9 @@
     emit_op_uint_le = gen_comp_op('emit_op_uint_le', c.LS)
     emit_op_uint_ge = gen_comp_op('emit_op_uint_ge', c.HS)
 
+    emit_op_ptr_eq = emit_op_instance_ptr_eq = emit_op_int_eq
+    emit_op_ptr_ne = emit_op_instance_ptr_ne = emit_op_int_ne
+
     def emit_op_int_is_true(self, op, arglocs):
         reg, res = arglocs
 
@@ -159,38 +162,65 @@
     def emit_op_gc_store(self, op, arglocs):
         value_loc, base_loc, ofs_loc, size_loc = arglocs
         scale = get_scale(size_loc.value)
-        self._write_to_mem(value_loc, base_loc, ofs_loc, imm(scale))
+        self._write_to_mem(value_loc, base_loc, ofs_loc, scale)
 
     def _emit_op_gc_load(self, op, arglocs):
         base_loc, ofs_loc, res_loc, nsize_loc = arglocs
         nsize = nsize_loc.value
         signed = (nsize < 0)
         scale = get_scale(abs(nsize))
-        self._load_from_mem(res_loc, base_loc, ofs_loc, imm(scale), signed)
+        self._load_from_mem(res_loc, base_loc, ofs_loc, scale, signed)
 
     emit_op_gc_load_i = _emit_op_gc_load
     emit_op_gc_load_r = _emit_op_gc_load
     emit_op_gc_load_f = _emit_op_gc_load
 
+    def emit_op_gc_store_indexed(self, op, arglocs):
+        value_loc, base_loc, index_loc, size_loc, ofs_loc = arglocs
+        assert index_loc.is_core_reg()
+        # add the base offset
+        if ofs_loc.value > 0:
+            self.mc.ADD_ri(r.ip0.value, index_loc.value, ofs_loc.value)
+            index_loc = r.ip0
+        scale = get_scale(size_loc.value)
+        self._write_to_mem(value_loc, base_loc, index_loc, scale)
+
+    def _emit_op_gc_load_indexed(self, op, arglocs):
+        res_loc, base_loc, index_loc, nsize_loc, ofs_loc = arglocs
+        assert index_loc.is_core_reg()
+        nsize = nsize_loc.value
+        signed = (nsize < 0)
+        # add the base offset
+        if ofs_loc.value > 0:
+            self.mc.ADD_ri(r.ip0.value, index_loc.value, ofs_loc.value)
+            index_loc = r.ip0
+        #
+        scale = get_scale(abs(nsize))
+        self._load_from_mem(res_loc, base_loc, index_loc, scale, signed)
+
+    emit_op_gc_load_indexed_i = _emit_op_gc_load_indexed
+    emit_op_gc_load_indexed_r = _emit_op_gc_load_indexed
+    emit_op_gc_load_indexed_f = _emit_op_gc_load_indexed
+
     def _write_to_mem(self, value_loc, base_loc, ofs_loc, scale):
         # Write a value of size '1 << scale' at the address
         # 'base_ofs + ofs_loc'.  Note that 'scale' is not used to scale
         # the offset!
         assert base_loc.is_core_reg()
-        if scale.value == 3:
+        if scale == 3:
             # WORD size
             if ofs_loc.is_imm():
                 self.mc.STR_ri(value_loc.value, base_loc.value,
                                 ofs_loc.value)
             else:
-                self.mc.STR_rr(value_loc.value, base_loc.value,
-                                ofs_loc.value)
+                self.mc.STR_size_rr(3, value_loc.value, base_loc.value,
+                                    ofs_loc.value)
         else:
             if ofs_loc.is_imm():
-                self.mc.STR_size_ri(scale.value, value_loc.value, base_loc.value,
+                self.mc.STR_size_ri(scale, value_loc.value, base_loc.value,
                                      ofs_loc.value)
             else:
-                self.mc.STR_size_rr(scale.value, value_loc.value, base_loc.value,
+                self.mc.STR_size_rr(scale, value_loc.value, base_loc.value,
                                      ofs_loc.value)
 
     def _load_from_mem(self, res_loc, base_loc, ofs_loc, scale,
@@ -199,21 +229,54 @@
         # 'base_loc + ofs_loc'.  Note that 'scale' is not used to scale
         # the offset!
         #
-        if scale.value == 3:
+        if scale == 3:
             # WORD
             if ofs_loc.is_imm():
-                self.mc.LDR_ri(res_loc.value, base_loc.value,
-                                ofs_loc.value)
+                self.mc.LDR_ri(res_loc.value, base_loc.value, ofs_loc.value)
             else:
-                self.mc.LDR_rr(res_loc.value, base_loc.value,
-                                ofs_loc.value)
+                self.mc.LDR_rr(res_loc.value, base_loc.value, ofs_loc.value)
+            return
+        if scale == 2:
+            # 32bit int
+            if not signed:
+                if ofs_loc.is_imm():
+                    self.mc.LDR_uint32_ri(res_loc.value, base_loc.value,
+                                          ofs_loc.value)
+                else:
+                    self.mc.LDR_uint32_rr(res_loc.value, base_loc.value,
+                                          ofs_loc.value)
+            else:
+                if ofs_loc.is_imm():
+                    self.mc.LDRSW_ri(res_loc.value, base_loc.value,
+                                             ofs_loc.value)
+                else:
+                    self.mc.LDRSW_rr(res_loc.value, base_loc.value,
+                                             ofs_loc.value)
+            return
+        if scale == 1:
+            # short
+            if not signed:
+                if ofs_loc.is_imm():
+                    self.mc.LDRH_ri(res_loc.value, base_loc.value, ofs_loc.value)
+                else:
+                    self.mc.LDRH_rr(res_loc.value, base_loc.value, ofs_loc.value)
+            else:
+                if ofs_loc.is_imm():
+                    self.mc.LDRSH_ri(res_loc.value, base_loc.value, ofs_loc.value)
+                else:
+                    self.mc.LDRSH_rr(res_loc.value, base_loc.value, ofs_loc.value)
+            return
+        assert scale == 0
+        if not signed:
+            if ofs_loc.is_imm():
+                self.mc.LDRB_ri(res_loc.value, base_loc.value, ofs_loc.value)
+            else:
+                self.mc.LDRB_rr(res_loc.value, base_loc.value, ofs_loc.value)
         else:
             if ofs_loc.is_imm():
-                self.mc.LDR_size_ri(scale.value, res_loc.value, base_loc.value,
-                                    ofs_loc.value)
+                self.mc.LDRSB_ri(res_loc.value, base_loc.value, ofs_loc.value)
             else:
-                self.mc.LDR_size_rr(scale.value, res_loc.value, base_loc.value,
-                                    ofs_loc.value)
+                self.mc.LDRSB_rr(res_loc.value, base_loc.value, ofs_loc.value)
 
     # -------------------------------- guard --------------------------------
 
@@ -256,6 +319,63 @@
         self._emit_guard(guard_op, c.get_opposite_of(fcond), arglocs)
     emit_guard_op_guard_overflow = emit_guard_op_guard_false
 
+
+    def load_condition_into_cc(self, loc):
+        if not loc.is_core_reg():
+            if loc.is_stack():
+                self.regalloc_mov(loc, r.ip0)
+            else:
+                assert loc.is_imm()
+                self.mc.gen_load_int(r.ip0.value, loc.value)
+            loc = r.ip0
+        self.mc.CMP_ri(loc.value, 0)
+
+    def emit_op_guard_false(self, op, arglocs):
+        self.load_condition_into_cc(arglocs[0])
+        self._emit_guard(op, c.EQ, arglocs[1:])
+    emit_op_guard_isnull = emit_op_guard_false
+
+    def emit_op_guard_true(self, op, arglocs):
+        self.load_condition_into_cc(arglocs[0])
+        self._emit_guard(op, c.NE, arglocs[1:])
+    emit_op_guard_nonnull = emit_op_guard_true
+
+    def emit_op_guard_value(self, op, arglocs):
+        v0 = arglocs[0]
+        assert v0.is_core_reg() # can be also a float reg, but later
+        v1 = arglocs[1]
+        if v1.is_core_reg():
+            loc = v1
+        elif v1.is_imm():
+            self.mc.gen_load_int(r.ip0.value, v1.value)
+            loc = r.ip0
+        else:
+            assert v1.is_stack()
+            yyy
+        self.mc.CMP_rr(v0.value, loc.value)
+        self._emit_guard(op, c.EQ, arglocs[2:])
+
+    def emit_op_guard_class(self, op, arglocs):
+        offset = self.cpu.vtable_offset
+        assert offset is not None
+        self.mc.LDR_ri(r.ip0.value, arglocs[0].value, offset)
+        self.mc.gen_load_int(r.ip1.value, arglocs[1].value)
+        self.mc.CMP_rr(r.ip0.value, r.ip1.value)
+        self._emit_guard(op, c.EQ, arglocs[2:])
+
+    def emit_op_guard_nonnull_class(self, op, arglocs):
+        offset = self.cpu.vtable_offset
+        assert offset is not None
+        # XXX a bit obscure think about a better way
+        self.mc.MOVZ_r_u16(r.ip0.value, 1, 0)
+        self.mc.MOVZ_r_u16(r.ip1.value, 0, 0)
+        self.mc.CMP_ri(arglocs[0].value, 0)
+        self.mc.B_ofs_cond(4 * (4 + 2), c.EQ)
+        self.mc.LDR_ri(r.ip0.value, arglocs[0].value, offset)
+        self.mc.gen_load_int_full(r.ip1.value, arglocs[1].value)
+        self.mc.CMP_rr(r.ip0.value, r.ip1.value)
+        self._emit_guard(op, c.EQ, arglocs[2:])        
+
     # ----------------------------- call ------------------------------
 
     def _genop_call(self, op, arglocs):
@@ -296,17 +416,6 @@
             else:
                 cb.emit_no_collect()
 
-    def load_condition_into_cc(self, loc):
-        if not loc.is_core_reg():
-            assert loc.is_stack()
-            self.regalloc_mov(loc, r.ip0)
-            loc = r.ip0
-        self.mc.CMP_ri(loc.value, 0)
-
-    def emit_op_guard_false(self, op, arglocs):
-        self.load_condition_into_cc(arglocs[1])
-        self._emit_guard(op, c.EQ, arglocs)
-
     def emit_op_label(self, op, arglocs):
         pass
 
@@ -315,9 +424,9 @@
         assert isinstance(target_token, TargetToken)
         target = target_token._ll_loop_code
         if target_token in self.target_tokens_currently_compiling:
-            self.mc.B_ofs(target)
+            self.mc.B_ofs(target - self.mc.currpos())
         else:
-            self.mc.BL(target)
+            self.mc.B(target)
 
     def emit_op_finish(self, op, arglocs):
         base_ofs = self.cpu.get_baseofs_of_frame_field()
diff --git a/rpython/jit/backend/aarch64/regalloc.py b/rpython/jit/backend/aarch64/regalloc.py
--- a/rpython/jit/backend/aarch64/regalloc.py
+++ b/rpython/jit/backend/aarch64/regalloc.py
@@ -212,6 +212,8 @@
             var = op.getarg(i)
             if var is not None:  # xxx kludgy
                 self.possibly_free_var(var)
+        if op.is_guard():
+            self.possibly_free_vars(op.getfailargs())
 
     def possibly_free_vars(self, vars):
         for var in vars:
@@ -444,6 +446,34 @@
     prepare_op_gc_load_r = _prepare_op_gc_load
     prepare_op_gc_load_f = _prepare_op_gc_load
 
+    def prepare_op_gc_store_indexed(self, op):
+        boxes = op.getarglist()
+        base_loc = self.make_sure_var_in_reg(boxes[0], boxes)
+        value_loc = self.make_sure_var_in_reg(boxes[2], boxes)
+        index_loc = self.make_sure_var_in_reg(boxes[1], boxes)
+        assert boxes[3].getint() == 1    # scale
+        ofs = boxes[4].getint()
+        size = boxes[5].getint()
+        assert check_imm_arg(ofs)
+        return [value_loc, base_loc, index_loc, imm(size), imm(ofs)]
+
+    def _prepare_op_gc_load_indexed(self, op):
+        boxes = op.getarglist()
+        base_loc = self.make_sure_var_in_reg(boxes[0], boxes)
+        index_loc = self.make_sure_var_in_reg(boxes[1], boxes)
+        assert boxes[2].getint() == 1    # scale
+        ofs = boxes[3].getint()
+        nsize = boxes[4].getint()
+        assert check_imm_arg(ofs)
+        self.possibly_free_vars_for_op(op)
+        self.free_temp_vars()
+        res_loc = self.force_allocate_reg(op)
+        return [res_loc, base_loc, index_loc, imm(nsize), imm(ofs)]
+
+    prepare_op_gc_load_indexed_i = _prepare_op_gc_load_indexed
+    prepare_op_gc_load_indexed_r = _prepare_op_gc_load_indexed
+    prepare_op_gc_load_indexed_f = _prepare_op_gc_load_indexed
+
     # --------------------------------- call ----------------------------
 
     def _prepare_op_call(self, op):
@@ -607,8 +637,40 @@
         return self._guard_impl(guard_op), c.VC
     prepare_guard_op_guard_no_overflow = prepare_guard_op_guard_overflow
 
-    prepare_op_guard_true = _guard_impl
-    prepare_op_guard_false = _guard_impl
+    def guard_no_cc_impl(self, op):
+        # rare case of guard with no CC
+        arglocs = self._guard_impl(op)
+        return [self.loc(op.getarg(0))] + arglocs
+
+    prepare_op_guard_true = guard_no_cc_impl
+    prepare_op_guard_false = guard_no_cc_impl
+    prepare_op_guard_nonnull = guard_no_cc_impl
+    prepare_op_guard_isnull = guard_no_cc_impl
+
+    def prepare_op_guard_value(self, op):
+        arg = self.make_sure_var_in_reg(op.getarg(0))
+        op.getdescr().make_a_counter_per_value(op,
+            self.cpu.all_reg_indexes[arg.value])
+        l1 = self.loc(op.getarg(1))
+        imm_a1 = check_imm_box(op.getarg(1))
+        if not imm_a1:
+            l1 = self.make_sure_var_in_reg(op.getarg(1), [arg])
+        arglocs = self._guard_impl(op)
+        return [arg, l1] + arglocs
+
+    def prepare_op_guard_class(self, op):
+        assert not isinstance(op.getarg(0), Const)
+        x = self.make_sure_var_in_reg(op.getarg(0))
+        y_val = rffi.cast(lltype.Signed, op.getarg(1).getint())
+        arglocs = self._guard_impl(op)
+        return [x, imm(y_val)] + arglocs
+
+    prepare_op_guard_nonnull_class = prepare_op_guard_class
+    prepare_op_guard_gc_type = prepare_op_guard_class
+    prepare_op_guard_subclass = prepare_op_guard_class
+
+    prepare_op_ptr_eq = prepare_op_instance_ptr_eq = prepare_op_int_eq
+    prepare_op_ptr_ne = prepare_op_instance_ptr_ne = prepare_op_int_ne
 
     prepare_op_nursery_ptr_increment = prepare_op_int_add
     prepare_comp_op_int_add_ovf = prepare_int_ri

From pypy.commits at gmail.com  Wed May 15 05:58:57 2019
From: pypy.commits at gmail.com (fijal)
Date: Wed, 15 May 2019 02:58:57 -0700 (PDT)
Subject: [pypy-commit] pypy arm64: skip part of the test and add some
 missing ops
Message-ID: <5cdbe2e1.1c69fb81.f599.bb86@mx.google.com>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: arm64
Changeset: r96616:7ca6dad13e3e
Date: 2019-05-15 09:58 +0000
http://bitbucket.org/pypy/pypy/changeset/7ca6dad13e3e/

Log:	skip part of the test and add some missing ops

diff --git a/rpython/jit/backend/aarch64/codebuilder.py b/rpython/jit/backend/aarch64/codebuilder.py
--- a/rpython/jit/backend/aarch64/codebuilder.py
+++ b/rpython/jit/backend/aarch64/codebuilder.py
@@ -125,6 +125,14 @@
         base = 0b11111000011
         self.write32((base << 21) | (rm << 16) | (0b011010 << 10) | (rn << 5) | rt)
 
+    def LDR_uint32_rr(self, rt, rn, rm):
+        base = 0b10111000011
+        self.write32((base << 21) | (rm << 16) | (0b011010 << 10) | (rn << 5) | rt)        
+
+    def LDRH_rr(self, rt, rn, rm):
+        base = 0b01111000011
+        self.write32((base << 21) | (rm << 16) | (0b011010 << 10) | (rn << 5) | rt)
+
     def LDRB_rr(self, rt, rn, rm):
         base = 0b00111000011
         self.write32((base << 21) | (rm << 16) | (0b011010 << 10) | (rn << 5) | rt)
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -1013,15 +1013,17 @@
         vsdescr = self.cpu.interiorfielddescrof(A, 'vs')
         kdescr = self.cpu.interiorfielddescrof(A, 'k')
         pdescr = self.cpu.interiorfielddescrof(A, 'p')
-        self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, InputArgInt(3),
-                                                         boxfloat(1.5)],
-                               'void', descr=kdescr)
-        f = self.cpu.bh_getinteriorfield_gc_f(a_box.getref_base(), 3, kdescr)
-        assert longlong.getrealfloat(f) == 1.5
-        self.cpu.bh_setinteriorfield_gc_f(a_box.getref_base(), 3, longlong.getfloatstorage(2.5), kdescr)
-        r = self.execute_operation(rop.GETINTERIORFIELD_GC_F, [a_box, InputArgInt(3)],
-                                   'float', descr=kdescr)
-        assert longlong.getrealfloat(r) == 2.5
+        if self.cpu.supports_floats:
+
+            self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, InputArgInt(3),
+                                                             boxfloat(1.5)],
+                                   'void', descr=kdescr)
+            f = self.cpu.bh_getinteriorfield_gc_f(a_box.getref_base(), 3, kdescr)
+            assert longlong.getrealfloat(f) == 1.5
+            self.cpu.bh_setinteriorfield_gc_f(a_box.getref_base(), 3, longlong.getfloatstorage(2.5), kdescr)
+            r = self.execute_operation(rop.GETINTERIORFIELD_GC_F, [a_box, InputArgInt(3)],
+                                       'float', descr=kdescr)
+            assert longlong.getrealfloat(r) == 2.5
         #
         NUMBER_FIELDS = [('vs', lltype.Signed),
                          ('vu', lltype.Unsigned),

From pypy.commits at gmail.com  Wed May 15 06:32:35 2019
From: pypy.commits at gmail.com (fijal)
Date: Wed, 15 May 2019 03:32:35 -0700 (PDT)
Subject: [pypy-commit] pypy arm64: same as
Message-ID: <5cdbeac3.1c69fb81.71b6.9bf4@mx.google.com>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: arm64
Changeset: r96617:4ba2f313407e
Date: 2019-05-15 10:31 +0000
http://bitbucket.org/pypy/pypy/changeset/4ba2f313407e/

Log:	same as

diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py
--- a/rpython/jit/backend/aarch64/opassembler.py
+++ b/rpython/jit/backend/aarch64/opassembler.py
@@ -157,6 +157,17 @@
         self.mc.ADD_ri(value_loc.value, value_loc.value, 1)
         self.mc.STR_ri(value_loc.value, base_loc.value, 0)
 
+    def _genop_same_as(self, op, arglocs):
+        argloc, resloc = arglocs
+        if argloc is not resloc:
+            self.mov_loc_loc(argloc, resloc)
+
+    emit_op_same_as_i = _genop_same_as
+    emit_op_same_as_r = _genop_same_as
+    emit_op_same_as_f = _genop_same_as
+    emit_op_cast_ptr_to_int = _genop_same_as
+    emit_op_cast_int_to_ptr = _genop_same_as
+
     # -------------------------------- fields -------------------------------
 
     def emit_op_gc_store(self, op, arglocs):
diff --git a/rpython/jit/backend/aarch64/regalloc.py b/rpython/jit/backend/aarch64/regalloc.py
--- a/rpython/jit/backend/aarch64/regalloc.py
+++ b/rpython/jit/backend/aarch64/regalloc.py
@@ -675,6 +675,24 @@
     prepare_op_nursery_ptr_increment = prepare_op_int_add
     prepare_comp_op_int_add_ovf = prepare_int_ri
 
+    def _prepare_op_same_as(self, op):
+        arg = op.getarg(0)
+        imm_arg = check_imm_box(arg)
+        if imm_arg:
+            argloc = self.convert_to_imm(arg)
+        else:
+            argloc = self.make_sure_var_in_reg(arg)
+        self.possibly_free_vars_for_op(op)
+        self.free_temp_vars()
+        resloc = self.force_allocate_reg(op)
+        return [argloc, resloc]
+
+    prepare_op_cast_ptr_to_int = _prepare_op_same_as
+    prepare_op_cast_int_to_ptr = _prepare_op_same_as
+    prepare_op_same_as_i = _prepare_op_same_as
+    prepare_op_same_as_r = _prepare_op_same_as
+    prepare_op_same_as_f = _prepare_op_same_as
+
     def prepare_op_jump(self, op):
         assert self.jump_target_descr is None
         descr = op.getdescr()

From pypy.commits at gmail.com  Wed May 15 08:59:40 2019
From: pypy.commits at gmail.com (fijal)
Date: Wed, 15 May 2019 05:59:40 -0700 (PDT)
Subject: [pypy-commit] pypy arm64: some progress
Message-ID: <5cdc0d3c.1c69fb81.f8207.d8a3@mx.google.com>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: arm64
Changeset: r96618:98c1f7b37259
Date: 2019-05-15 12:58 +0000
http://bitbucket.org/pypy/pypy/changeset/98c1f7b37259/

Log:	some progress

diff --git a/rpython/jit/backend/aarch64/assembler.py b/rpython/jit/backend/aarch64/assembler.py
--- a/rpython/jit/backend/aarch64/assembler.py
+++ b/rpython/jit/backend/aarch64/assembler.py
@@ -628,8 +628,10 @@
                 regalloc.possibly_free_vars_for_op(op)
             elif not we_are_translated() and op.getopnum() == rop.FORCE_SPILL:
                 regalloc.prepare_force_spill(op)
-            elif i < len(operations) - 1 and (regalloc.next_op_can_accept_cc(operations, i) or
-                                              operations[i].is_ovf()):
+            elif ((rop.returns_bool_result(opnum) or op.is_ovf()) and 
+                  i < len(operations) - 1 and
+                  regalloc.next_op_can_accept_cc(operations, i) or
+                                               operations[i].is_ovf()):
                 if operations[i].is_ovf():
                     assert operations[i + 1].getopnum() in [rop.GUARD_OVERFLOW,
                                                             rop.GUARD_NO_OVERFLOW]
@@ -639,6 +641,8 @@
                 if arglocs is not None:
                     asm_guard_operations[guard_num](self, guard_op, fcond, arglocs)
                 regalloc.next_instruction() # advance one more
+                regalloc.possibly_free_vars(guard_op.getfailargs())
+                regalloc.possibly_free_vars_for_op(guard_op)
             else:
                 arglocs = regalloc_operations[opnum](regalloc, op)
                 if arglocs is not None:
diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py
--- a/rpython/jit/backend/aarch64/opassembler.py
+++ b/rpython/jit/backend/aarch64/opassembler.py
@@ -111,6 +111,14 @@
         self.emit_int_comp_op(op, arglocs[0], arglocs[1])
         return c.LE
 
+    def emit_comp_op_int_gt(self, op, arglocs):
+        self.emit_int_comp_op(op, arglocs[0], arglocs[1])
+        return c.GT
+
+    def emit_comp_op_int_ge(self, op, arglocs):
+        self.emit_int_comp_op(op, arglocs[0], arglocs[1])
+        return c.GE
+
     def emit_comp_op_int_eq(self, op, arglocs):
         self.emit_int_comp_op(op, arglocs[0], arglocs[1])
         return c.EQ
@@ -168,6 +176,11 @@
     emit_op_cast_ptr_to_int = _genop_same_as
     emit_op_cast_int_to_ptr = _genop_same_as
 
+    def emit_op_load_from_gc_table(self, op, arglocs):
+        res_loc, = arglocs
+        index = op.getarg(0).getint()
+        self.load_from_gc_table(res_loc.value, index)
+
     # -------------------------------- fields -------------------------------
 
     def emit_op_gc_store(self, op, arglocs):
diff --git a/rpython/jit/backend/aarch64/regalloc.py b/rpython/jit/backend/aarch64/regalloc.py
--- a/rpython/jit/backend/aarch64/regalloc.py
+++ b/rpython/jit/backend/aarch64/regalloc.py
@@ -207,6 +207,12 @@
         else:
             self.rm.possibly_free_var(var)
 
+    def force_spill_var(self, var):
+        if var.type == FLOAT:
+            self.vfprm.force_spill_var(var)
+        else:
+            self.rm.force_spill_var(var)
+
     def possibly_free_vars_for_op(self, op):
         for i in range(op.numargs()):
             var = op.getarg(i)
@@ -385,6 +391,8 @@
     prepare_comp_op_int_lt = prepare_int_cmp
     prepare_comp_op_int_le = prepare_int_cmp
     prepare_comp_op_int_eq = prepare_int_cmp
+    prepare_comp_op_int_ge = prepare_int_cmp
+    prepare_comp_op_int_gt = prepare_int_cmp
 
     def prepare_op_int_le(self, op):
         return self.prepare_int_cmp(op, False)
@@ -693,6 +701,10 @@
     prepare_op_same_as_r = _prepare_op_same_as
     prepare_op_same_as_f = _prepare_op_same_as
 
+    def prepare_op_load_from_gc_table(self, op):
+        resloc = self.force_allocate_reg(op)
+        return [resloc]
+
     def prepare_op_jump(self, op):
         assert self.jump_target_descr is None
         descr = op.getdescr()

From pypy.commits at gmail.com  Wed May 15 09:51:12 2019
From: pypy.commits at gmail.com (fijal)
Date: Wed, 15 May 2019 06:51:12 -0700 (PDT)
Subject: [pypy-commit] pypy arm64: general progress
Message-ID: <5cdc1950.1c69fb81.97832.86b2@mx.google.com>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: arm64
Changeset: r96619:85965b5b4b41
Date: 2019-05-15 13:50 +0000
http://bitbucket.org/pypy/pypy/changeset/85965b5b4b41/

Log:	general progress

diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py
--- a/rpython/jit/backend/aarch64/opassembler.py
+++ b/rpython/jit/backend/aarch64/opassembler.py
@@ -123,6 +123,26 @@
         self.emit_int_comp_op(op, arglocs[0], arglocs[1])
         return c.EQ
 
+    def emit_comp_op_int_ne(self, op, arglocs):
+        self.emit_int_comp_op(op, arglocs[0], arglocs[1])
+        return c.NE
+
+    def emit_comp_op_uint_lt(self, op, arglocs):
+        self.emit_int_comp_op(op, arglocs[0], arglocs[1])
+        return c.LO
+
+    def emit_comp_op_uint_le(self, op, arglocs):
+        self.emit_int_comp_op(op, arglocs[0], arglocs[1])
+        return c.LS
+
+    def emit_comp_op_uint_gt(self, op, arglocs):
+        self.emit_int_comp_op(op, arglocs[0], arglocs[1])
+        return c.HI
+
+    def emit_comp_op_uint_ge(self, op, arglocs):
+        self.emit_int_comp_op(op, arglocs[0], arglocs[1])
+        return c.HS
+
     emit_op_int_lt = gen_comp_op('emit_op_int_lt', c.LT)
     emit_op_int_le = gen_comp_op('emit_op_int_le', c.LE)
     emit_op_int_gt = gen_comp_op('emit_op_int_gt', c.GT)
@@ -144,12 +164,20 @@
         self.mc.CMP_ri(reg.value, 0)
         self.mc.CSET_r_flag(res.value, c.EQ)
 
+    def emit_comp_op_int_is_true(self, op, arglocs):
+        self.mc.CMP_ri(arglocs[0].value, 0)
+        return c.NE
+
     def emit_op_int_is_zero(self, op, arglocs):
         reg, res = arglocs
 
         self.mc.CMP_ri(reg.value, 0)
         self.mc.CSET_r_flag(res.value, c.NE)
 
+    def emit_comp_op_int_is_zero(self, op, arglocs):
+        self.mc.CMP_ri(arglocs[0].value, 0)
+        return c.EQ
+
     def emit_op_int_neg(self, op, arglocs):
         reg, res = arglocs
         self.mc.SUB_rr_shifted(res.value, r.xzr.value, reg.value)
@@ -181,6 +209,15 @@
         index = op.getarg(0).getint()
         self.load_from_gc_table(res_loc.value, index)
 
+    def emit_op_debug_merge_point(self, op, arglocs):
+        pass
+    
+    emit_op_jit_debug = emit_op_debug_merge_point
+    emit_op_keepalive = emit_op_debug_merge_point
+    emit_op_enter_portal_frame = emit_op_debug_merge_point
+    emit_op_leave_portal_frame = emit_op_debug_merge_point
+
+
     # -------------------------------- fields -------------------------------
 
     def emit_op_gc_store(self, op, arglocs):
diff --git a/rpython/jit/backend/aarch64/regalloc.py b/rpython/jit/backend/aarch64/regalloc.py
--- a/rpython/jit/backend/aarch64/regalloc.py
+++ b/rpython/jit/backend/aarch64/regalloc.py
@@ -14,7 +14,7 @@
         get_scale
 from rpython.rtyper.lltypesystem import lltype, rffi, rstr, llmemory
 from rpython.jit.backend.aarch64 import registers as r
-from rpython.jit.backend.arm.jump import remap_frame_layout_mixed
+from rpython.jit.backend.aarch64.jump import remap_frame_layout_mixed
 from rpython.jit.backend.aarch64.locations import imm
 from rpython.jit.backend.llsupport.gcmap import allocate_gcmap
 from rpython.jit.backend.llsupport.descr import CallDescr
@@ -303,6 +303,11 @@
         self.free_temp_vars()
         return [base_loc, value_loc]
 
+    def void(self, op):
+        return []
+
+    prepare_op_jit_debug = void
+
     def prepare_int_ri(self, op, res_in_cc):
         boxes = op.getarglist()
         a0, a1 = boxes
@@ -390,9 +395,14 @@
 
     prepare_comp_op_int_lt = prepare_int_cmp
     prepare_comp_op_int_le = prepare_int_cmp
-    prepare_comp_op_int_eq = prepare_int_cmp
     prepare_comp_op_int_ge = prepare_int_cmp
     prepare_comp_op_int_gt = prepare_int_cmp
+    prepare_comp_op_int_ne = prepare_int_cmp
+    prepare_comp_op_int_eq = prepare_int_cmp
+    prepare_comp_op_uint_lt = prepare_int_cmp
+    prepare_comp_op_uint_le = prepare_int_cmp
+    prepare_comp_op_uint_ge = prepare_int_cmp
+    prepare_comp_op_uint_gt = prepare_int_cmp
 
     def prepare_op_int_le(self, op):
         return self.prepare_int_cmp(op, False)
@@ -420,6 +430,15 @@
     prepare_op_int_neg = prepare_unary
     prepare_op_int_invert = prepare_unary
 
+    def prepare_comp_unary(self, op, res_in_cc):
+        a0 = op.getarg(0)
+        assert not isinstance(a0, Const)
+        reg = self.make_sure_var_in_reg(a0)
+        return [reg]
+
+    prepare_comp_op_int_is_true = prepare_comp_unary
+    prepare_comp_op_int_is_zero = prepare_comp_unary        
+
     # --------------------------------- fields --------------------------
 
     def prepare_op_gc_store(self, op):

From pypy.commits at gmail.com  Wed May 15 10:52:48 2019
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 15 May 2019 07:52:48 -0700 (PDT)
Subject: [pypy-commit] pypy optimizeopt-cleanup: simplify getrawptrinfo()
Message-ID: <5cdc27c0.1c69fb81.602b9.551c@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: optimizeopt-cleanup
Changeset: r96620:1dd36a7a4387
Date: 2019-05-13 13:53 +0100
http://bitbucket.org/pypy/pypy/changeset/1dd36a7a4387/

Log:	simplify getrawptrinfo()

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -859,10 +859,6 @@
     if isinstance(fw, IntBound):
         return None
     if fw is not None:
-        if isinstance(fw, AbstractRawPtrInfo):
-            return fw
-        fw = RawStructPtrInfo()
-        op.set_forwarded(fw)
         assert isinstance(fw, AbstractRawPtrInfo)
         return fw
     return None
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -851,4 +851,3 @@
 
 dispatch_opt = make_dispatcher_method(Optimizer, 'optimize_',
         default=Optimizer.optimize_default)
-

From pypy.commits at gmail.com  Wed May 15 10:52:50 2019
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 15 May 2019 07:52:50 -0700 (PDT)
Subject: [pypy-commit] pypy optimizeopt-cleanup: Replace method
 Optimization.getptrinfo() with function getptrinfo()
Message-ID: <5cdc27c2.1c69fb81.c928d.c4f2@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: optimizeopt-cleanup
Changeset: r96621:4d02009079d2
Date: 2019-05-15 15:51 +0100
http://bitbucket.org/pypy/pypy/changeset/4d02009079d2/

Log:	Replace method Optimization.getptrinfo() with function getptrinfo()

diff --git a/rpython/jit/metainterp/optimizeopt/bridgeopt.py b/rpython/jit/metainterp/optimizeopt/bridgeopt.py
--- a/rpython/jit/metainterp/optimizeopt/bridgeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/bridgeopt.py
@@ -3,6 +3,7 @@
 
 from rpython.jit.metainterp import resumecode
 from rpython.jit.metainterp.history import Const, ConstInt, CONST_NULL
+from .info import getptrinfo
 
 
 # adds the following sections at the end of the resume code:
@@ -75,7 +76,7 @@
     for box in liveboxes:
         if box is None or box.type != "r":
             continue
-        info = optimizer.getptrinfo(box)
+        info = getptrinfo(box)
         known_class = info is not None and info.get_known_class(optimizer.cpu) is not None
         bitfield <<= 1
         bitfield |= known_class
diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -61,36 +61,28 @@
             info.produce_short_preamble_ops(structbox, descr, index, optimizer,
                                             shortboxes)
 
-    def possible_aliasing(self, optheap, opinfo):
+    def possible_aliasing(self, opinfo):
         # If lazy_set is set and contains a setfield on a different
         # structvalue, then we are annoyed, because it may point to either
         # the same or a different structure at runtime.
         # XXX constants?
         return (self._lazy_set is not None
-                and (not optheap.getptrinfo(
-                    self._lazy_set.getarg(0)).same_info(opinfo)))
+            and not info.getptrinfo(self._lazy_set.getarg(0)).same_info(opinfo))
 
     def do_setfield(self, optheap, op):
         # Update the state with the SETFIELD_GC/SETARRAYITEM_GC operation 'op'.
         structinfo = optheap.ensure_ptr_info_arg0(op)
         arg1 = get_box_replacement(self._get_rhs_from_set_op(op))
-        if self.possible_aliasing(optheap, structinfo):
+        if self.possible_aliasing(structinfo):
             self.force_lazy_set(optheap, op.getdescr())
-            assert not self.possible_aliasing(optheap, structinfo)
+            assert not self.possible_aliasing(structinfo)
         cached_field = self._getfield(structinfo, op.getdescr(), optheap, False)
         if cached_field is not None:
             cached_field = cached_field.get_box_replacement()
 
-        # Hack to ensure constants are imported from the preamble
-        # XXX no longer necessary?
-        #if cached_fieldvalue and fieldvalue.is_constant():
-        #    optheap.optimizer.ensure_imported(cached_fieldvalue)
-        #    cached_fieldvalue = self._cached_fields.get(structvalue, None)
-
         if not cached_field or not cached_field.same_box(arg1):
             # common case: store the 'op' as lazy_set
             self._lazy_set = op
-
         else:
             # this is the case where the pending setfield ends up
             # storing precisely the value that is already there,
@@ -105,7 +97,7 @@
 
     def getfield_from_cache(self, optheap, opinfo, descr):
         # Returns the up-to-date field's value, or None if not cached.
-        if self.possible_aliasing(optheap, opinfo):
+        if self.possible_aliasing(opinfo):
             self.force_lazy_set(optheap, descr)
         if self._lazy_set is not None:
             op = self._lazy_set
@@ -526,7 +518,7 @@
                 # guards' resume data is that of a virtual object that is stored
                 # into a field of a non-virtual object.  Here, 'op' in either
                 # SETFIELD_GC or SETARRAYITEM_GC.
-                opinfo = self.getptrinfo(op.getarg(0))
+                opinfo = info.getptrinfo(op.getarg(0))
                 assert not opinfo.is_virtual()      # it must be a non-virtual
                 if self.optimizer.is_virtual(op.getarg(2)):
                     pendingfields.append(op)
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -1,10 +1,11 @@
 
-from rpython.rlib.objectmodel import specialize, we_are_translated, compute_hash
-from rpython.jit.metainterp.resoperation import AbstractValue, ResOperation,\
-     rop, OpHelpers
-from rpython.jit.metainterp.history import ConstInt, Const
-from rpython.rtyper.lltypesystem import lltype, llmemory
-from rpython.jit.metainterp.optimizeopt.rawbuffer import RawBuffer, InvalidRawOperation
+from rpython.rlib.objectmodel import specialize, compute_hash
+from rpython.jit.metainterp.resoperation import (
+    AbstractValue, ResOperation, rop, OpHelpers)
+from rpython.jit.metainterp.history import ConstInt, ConstPtr, Const
+from rpython.rtyper.lltypesystem import lltype
+from rpython.jit.metainterp.optimizeopt.rawbuffer import (
+    RawBuffer, InvalidRawOperation)
 from rpython.jit.metainterp.executor import execute
 from rpython.jit.metainterp.optimize import InvalidLoop
 from .util import get_box_replacement
@@ -84,7 +85,7 @@
 
     @specialize.arg(2)
     def get_constant_string_spec(self, string_optimizer, mode):
-        return None # can't be constant
+        return None  # can't be constant
 
 class NonNullPtrInfo(PtrInfo):
     _attrs_ = ('last_guard_pos',)
@@ -161,14 +162,14 @@
     def is_virtual(self):
         return self._is_virtual
 
-    def _visitor_walk_recursive(self, op, visitor, optimizer):
+    def _visitor_walk_recursive(self, op, visitor):
         raise NotImplementedError("abstract")
 
-    def visitor_walk_recursive(self, instbox, visitor, optimizer):
+    def visitor_walk_recursive(self, instbox, visitor):
         instbox = instbox.get_box_replacement()
         if visitor.already_seen_virtual(instbox):
             return
-        return self._visitor_walk_recursive(instbox, visitor, optimizer)
+        return self._visitor_walk_recursive(instbox, visitor)
 
 
 class AbstractStructPtrInfo(AbstractVirtualPtrInfo):
@@ -181,7 +182,7 @@
             self.descr = descr
             self._fields = [None] * len(descr.get_all_fielddescrs())
         if index >= len(self._fields):
-            self.descr = descr # a more precise descr
+            self.descr = descr  # a more precise descr
             # we found out a subclass with more fields
             extra_len = len(descr.get_all_fielddescrs()) - len(self._fields)
             self._fields = self._fields + [None] * extra_len
@@ -232,14 +233,14 @@
         rec[self] = None
         for i, fldbox in enumerate(self._fields):
             if fldbox is not None:
-                info = optforce.optimizer.getptrinfo(fldbox)
+                info = getptrinfo(fldbox)
                 if info is not None:
                     fldbox = info.force_at_the_end_of_preamble(fldbox, optforce,
                                                                rec)
                     self._fields[i] = fldbox
         return op
 
-    def _visitor_walk_recursive(self, instbox, visitor, optimizer):
+    def _visitor_walk_recursive(self, instbox, visitor):
         lst = self.descr.get_all_fielddescrs()
         assert self.is_virtual()
         visitor.register_virtual_fields(
@@ -247,9 +248,9 @@
         for i in range(len(lst)):
             op = self._fields[i]
             if op:
-                fieldinfo = optimizer.getptrinfo(op)
+                fieldinfo = getptrinfo(op)
                 if fieldinfo and fieldinfo.is_virtual():
-                    fieldinfo.visitor_walk_recursive(op, visitor, optimizer)
+                    fieldinfo.visitor_walk_recursive(op, visitor)
 
     def produce_short_preamble_ops(self, structbox, fielddescr, index, optimizer,
                                    shortboxes):
@@ -289,7 +290,7 @@
             if op.is_constant():
                 pass            # it is a constant value: ok
             else:
-                fieldinfo = optimizer.getptrinfo(op)
+                fieldinfo = getptrinfo(op)
                 if fieldinfo and fieldinfo.is_virtual():
                     # recursive check
                     if memo is None:
@@ -434,7 +435,7 @@
                               [op, ConstInt(offset), itembox], descr=descr)
             optforce.emit_extra(setfield_op)
 
-    def _visitor_walk_recursive(self, op, visitor, optimizer):
+    def _visitor_walk_recursive(self, op, visitor):
         itemboxes = [get_box_replacement(box)
                      for box in self._get_buffer().values]
         visitor.register_virtual_fields(op, itemboxes)
@@ -464,7 +465,7 @@
         return self.parent is not None
 
     def getitem_raw(self, offset, itemsize, descr):
-        return self.parent.getitem_raw(self.offset+offset, itemsize, descr)
+        return self.parent.getitem_raw(self.offset + offset, itemsize, descr)
 
     def setitem_raw(self, offset, itemsize, descr, itemop):
         self.parent.setitem_raw(self.offset + offset, itemsize, descr, itemop)
@@ -474,11 +475,11 @@
             self.parent._force_elements(op, optforce, descr)
         self.parent = None
 
-    def _visitor_walk_recursive(self, op, visitor, optimizer):
+    def _visitor_walk_recursive(self, op, visitor):
         source_op = get_box_replacement(op.getarg(0))
         visitor.register_virtual_fields(op, [source_op])
         if self.parent.is_virtual():
-            self.parent.visitor_walk_recursive(source_op, visitor, optimizer)
+            self.parent.visitor_walk_recursive(source_op, visitor)
 
     @specialize.argtype(1)
     def visitor_dispatch_virtual_type(self, visitor):
@@ -569,16 +570,15 @@
     def getlength(self):
         return self.length
 
-    def _visitor_walk_recursive(self, instbox, visitor, optimizer):
+    def _visitor_walk_recursive(self, instbox, visitor):
         itemops = [get_box_replacement(item) for item in self._items]
         visitor.register_virtual_fields(instbox, itemops)
         for i in range(self.getlength()):
             itemop = self._items[i]
-            if (itemop is not None and
-                not isinstance(itemop, Const)):
-                ptrinfo = optimizer.getptrinfo(itemop)
+            if (itemop is not None and not isinstance(itemop, Const)):
+                ptrinfo = getptrinfo(itemop)
                 if ptrinfo and ptrinfo.is_virtual():
-                    ptrinfo.visitor_walk_recursive(itemop, visitor, optimizer)
+                    ptrinfo.visitor_walk_recursive(itemop, visitor)
 
     @specialize.argtype(1)
     def visitor_dispatch_virtual_type(self, visitor):
@@ -608,7 +608,7 @@
         rec[self] = None
         for i, fldbox in enumerate(self._items):
             if fldbox is not None:
-                info = optforce.getptrinfo(fldbox)
+                info = getptrinfo(fldbox)
                 if info is not None:
                     fldbox = info.force_at_the_end_of_preamble(fldbox, optforce,
                                                                rec)
@@ -666,7 +666,7 @@
                     # if it does, we would need a fix here
                 i += 1
 
-    def _visitor_walk_recursive(self, instbox, visitor, optimizer):
+    def _visitor_walk_recursive(self, instbox, visitor):
         itemops = [get_box_replacement(item) for item in self._items]
         visitor.register_virtual_fields(instbox, itemops)
         fielddescrs = self.descr.get_all_fielddescrs()
@@ -675,10 +675,9 @@
             for fielddescr in fielddescrs:
                 itemop = self._items[i]
                 if (itemop is not None and not isinstance(itemop, Const)):
-                    ptrinfo = optimizer.getptrinfo(itemop)
+                    ptrinfo = getptrinfo(itemop)
                     if ptrinfo and ptrinfo.is_virtual():
-                        ptrinfo.visitor_walk_recursive(itemop, visitor,
-                                                       optimizer)
+                        ptrinfo.visitor_walk_recursive(itemop, visitor)
                 i += 1
 
     @specialize.argtype(1)
@@ -862,3 +861,20 @@
         assert isinstance(fw, AbstractRawPtrInfo)
         return fw
     return None
+
+def getptrinfo(op):
+    if op.type == 'i':
+        return getrawptrinfo(op)
+    elif op.type == 'f':
+        return None
+    assert op.type == 'r'
+    op = get_box_replacement(op)
+    assert op.type == 'r'
+    if isinstance(op, ConstPtr):
+        return ConstPtrInfo(op)
+    fw = op.get_forwarded()
+    if fw is not None:
+        assert isinstance(fw, PtrInfo)
+        return fw
+    return None
+
diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -7,6 +7,7 @@
     CONST_0)
 from rpython.jit.metainterp.optimizeopt.util import (
     make_dispatcher_method, get_box_replacement)
+from .info import getptrinfo
 from rpython.jit.metainterp.resoperation import rop
 from rpython.jit.metainterp.optimizeopt import vstring
 from rpython.jit.codewriter.effectinfo import EffectInfo
@@ -443,7 +444,7 @@
 
     def postprocess_STRLEN(self, op):
         self.make_nonnull_str(op.getarg(0), vstring.mode_string)
-        array = self.getptrinfo(op.getarg(0))
+        array = getptrinfo(op.getarg(0))
         self.optimizer.setintbound(op, array.getlenbound(vstring.mode_string))
 
     def optimize_UNICODELEN(self, op):
@@ -451,7 +452,7 @@
 
     def postprocess_UNICODELEN(self, op):
         self.make_nonnull_str(op.getarg(0), vstring.mode_unicode)
-        array = self.getptrinfo(op.getarg(0))
+        array = getptrinfo(op.getarg(0))
         self.optimizer.setintbound(op, array.getlenbound(vstring.mode_unicode))
 
     def optimize_STRGETITEM(self, op):
@@ -459,7 +460,7 @@
 
     def postprocess_STRGETITEM(self, op):
         v1 = self.getintbound(op)
-        v2 = self.getptrinfo(op.getarg(0))
+        v2 = getptrinfo(op.getarg(0))
         intbound = self.getintbound(op.getarg(1))
         if (intbound.has_lower and v2 is not None and
             v2.getlenbound(vstring.mode_string) is not None):
@@ -524,7 +525,7 @@
     def postprocess_UNICODEGETITEM(self, op):
         b1 = self.getintbound(op)
         b1.make_ge(IntLowerBound(0))
-        v2 = self.getptrinfo(op.getarg(0))
+        v2 = getptrinfo(op.getarg(0))
         intbound = self.getintbound(op.getarg(1))
         if (intbound.has_lower and v2 is not None and
             v2.getlenbound(vstring.mode_unicode) is not None):
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -1,7 +1,7 @@
 from rpython.jit.metainterp import jitprof, resume, compile
 from rpython.jit.metainterp.executor import execute_nonspec_const
 from rpython.jit.metainterp.history import (
-    Const, ConstInt, ConstPtr, CONST_NULL, new_ref_dict)
+    Const, ConstInt, CONST_NULL, new_ref_dict)
 from rpython.jit.metainterp.optimizeopt.intutils import (
     IntBound, ConstIntBound, MININT, MAXINT, IntUnbounded)
 from rpython.jit.metainterp.optimizeopt.util import (
@@ -10,7 +10,7 @@
     deserialize_optimizer_knowledge)
 from rpython.jit.metainterp.resoperation import (
     rop, AbstractResOp, GuardResOp, OpHelpers)
-from .info import getrawptrinfo
+from .info import getrawptrinfo, getptrinfo
 from rpython.jit.metainterp.optimizeopt import info
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.rlib.objectmodel import specialize, we_are_translated
@@ -111,7 +111,7 @@
 
     def getnullness(self, op):
         if op.type == 'r' or self.is_raw_ptr(op):
-            ptrinfo = self.getptrinfo(op)
+            ptrinfo = getptrinfo(op)
             if ptrinfo is None:
                 return info.INFO_UNKNOWN
             return ptrinfo.getnullness()
@@ -120,7 +120,7 @@
         assert False
 
     def make_constant_class(self, op, class_const, update_last_guard=True):
-        op = get_box_replacement(op)
+        op = op.get_box_replacement()
         opinfo = op.get_forwarded()
         if isinstance(opinfo, info.InstancePtrInfo):
             opinfo._known_class = class_const
@@ -136,22 +136,6 @@
             opinfo.mark_last_guard(self.optimizer)
         return opinfo
 
-    def getptrinfo(self, op):
-        if op.type == 'i':
-            return getrawptrinfo(op)
-        elif op.type == 'f':
-            return None
-        assert op.type == 'r'
-        op = get_box_replacement(op)
-        assert op.type == 'r'
-        if isinstance(op, ConstPtr):
-            return info.ConstPtrInfo(op)
-        fw = op.get_forwarded()
-        if fw is not None:
-            assert isinstance(fw, info.PtrInfo)
-            return fw
-        return None
-
     def is_raw_ptr(self, op):
         fw = get_box_replacement(op).get_forwarded()
         if isinstance(fw, info.AbstractRawPtrInfo):
@@ -303,7 +287,7 @@
 
     def force_box_for_end_of_preamble(self, box):
         if box.type == 'r':
-            info = self.getptrinfo(box)
+            info = getptrinfo(box)
             if info is not None and info.is_virtual():
                 rec = {}
                 return info.force_at_the_end_of_preamble(box,
@@ -326,10 +310,10 @@
 
     def getinfo(self, op):
         if op.type == 'r':
-            return self.getptrinfo(op)
+            return getptrinfo(op)
         elif op.type == 'i':
             if self.is_raw_ptr(op):
-                return self.getptrinfo(op)
+                return getptrinfo(op)
             return self.getintbound(op)
         elif op.type == 'f':
             if get_box_replacement(op).is_constant():
@@ -424,7 +408,7 @@
             return
         if box.type == 'r' and box.get_forwarded() is not None:
             opinfo = box.get_forwarded()
-            opinfo.copy_fields_to_const(self.getptrinfo(constbox), self.optheap)
+            opinfo.copy_fields_to_const(getptrinfo(constbox), self.optheap)
         box.set_forwarded(constbox)
 
     def make_constant_int(self, box, intvalue):
@@ -833,13 +817,8 @@
                 <= expected_class.subclassrange_max)
 
     def is_virtual(self, op):
-        if op.type == 'r':
-            opinfo = self.getptrinfo(op)
-            return opinfo is not None and opinfo.is_virtual()
-        if op.type == 'i':
-            opinfo = getrawptrinfo(op)
-            return opinfo is not None and opinfo.is_virtual()
-        return False
+        opinfo = getptrinfo(op)
+        return opinfo is not None and opinfo.is_virtual()
 
     # These are typically removed already by OptRewrite, but it can be
     # dissabled and unrolling emits some SAME_AS ops to setup the
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -7,14 +7,13 @@
 from rpython.jit.metainterp.optimizeopt.intutils import IntBound
 from rpython.jit.metainterp.optimizeopt.optimizer import (
     Optimization, OptimizationResult, REMOVED, CONST_0, CONST_1)
-from rpython.jit.metainterp.optimizeopt.info import INFO_NONNULL, INFO_NULL
+from rpython.jit.metainterp.optimizeopt.info import (
+    INFO_NONNULL, INFO_NULL, getptrinfo)
 from rpython.jit.metainterp.optimizeopt.util import (
     _findall, make_dispatcher_method, get_box_replacement)
-from rpython.jit.metainterp.resoperation import rop, ResOperation, opclasses,\
-     OpHelpers
+from rpython.jit.metainterp.resoperation import (
+    rop, ResOperation, opclasses, OpHelpers)
 from rpython.rlib.rarithmetic import highest_bit
-from rpython.rtyper.lltypesystem import llmemory
-from rpython.rtyper import rclass
 import math
 
 
@@ -69,7 +68,6 @@
                 return True
         return False
 
-
     def find_rewritable_bool(self, op):
         oldopnum = op.boolinverse
         arg0 = op.getarg(0)
@@ -79,7 +77,7 @@
             if self.try_boolinvers(op, top):
                 return True
 
-        oldopnum = op.boolreflex # FIXME: add INT_ADD, INT_MUL
+        oldopnum = op.boolreflex  # FIXME: add INT_ADD, INT_MUL
         if oldopnum != -1:
             top = ResOperation(oldopnum, [arg1, arg0])
             oldop = self.get_pure_result(top)
@@ -344,7 +342,7 @@
         return self.emit(op)
 
     def optimize_GUARD_ISNULL(self, op):
-        info = self.getptrinfo(op.getarg(0))
+        info = getptrinfo(op.getarg(0))
         if info is not None:
             if info.is_null():
                 return
@@ -358,7 +356,7 @@
         self.make_constant(op.getarg(0), CONST_NULL)
 
     def optimize_GUARD_IS_OBJECT(self, op):
-        info = self.getptrinfo(op.getarg(0))
+        info = getptrinfo(op.getarg(0))
         if info and info.is_constant():
             if info.is_null():
                 raise InvalidLoop("A GUARD_IS_OBJECT(NULL) found")
@@ -374,7 +372,7 @@
         return self.emit(op)
 
     def optimize_GUARD_GC_TYPE(self, op):
-        info = self.getptrinfo(op.getarg(0))
+        info = getptrinfo(op.getarg(0))
         if info and info.is_constant():
             c = get_box_replacement(op.getarg(0))
             tid = self.optimizer.cpu.get_actual_typeid(c.getref_base())
@@ -388,7 +386,7 @@
         return self.emit(op)
 
     def optimize_GUARD_SUBCLASS(self, op):
-        info = self.getptrinfo(op.getarg(0))
+        info = getptrinfo(op.getarg(0))
         optimizer = self.optimizer
         # must raise 'InvalidLoop' in all cases where 'info' shows the
         # class cannot possibly match (see test_issue2926)
@@ -427,7 +425,7 @@
         return self.emit(op)
 
     def optimize_GUARD_NONNULL(self, op):
-        opinfo = self.getptrinfo(op.getarg(0))
+        opinfo = getptrinfo(op.getarg(0))
         if opinfo is not None:
             if opinfo.is_nonnull():
                 return
@@ -439,12 +437,12 @@
 
     def postprocess_GUARD_NONNULL(self, op):
         self.make_nonnull(op.getarg(0))
-        self.getptrinfo(op.getarg(0)).mark_last_guard(self.optimizer)
+        getptrinfo(op.getarg(0)).mark_last_guard(self.optimizer)
 
     def optimize_GUARD_VALUE(self, op):
         arg0 = op.getarg(0)
         if arg0.type == 'r':
-            info = self.getptrinfo(arg0)
+            info = getptrinfo(arg0)
             if info:
                 if info.is_virtual():
                     raise InvalidLoop("promote of a virtual")
@@ -492,8 +490,8 @@
                                   'always fail' % r)
         descr = compile.ResumeGuardDescr()
         op = old_guard_op.copy_and_change(rop.GUARD_VALUE,
-                         args = [old_guard_op.getarg(0), op.getarg(1)],
-                         descr = descr)
+                         args=[old_guard_op.getarg(0), op.getarg(1)],
+                         descr=descr)
         # Note: we give explicitly a new descr for 'op'; this is why the
         # old descr must not be ResumeAtPositionDescr (checked above).
         # Better-safe-than-sorry but it should never occur: we should
@@ -532,7 +530,7 @@
         self.make_nonnull(op.getarg(0))
 
     def optimize_RECORD_EXACT_CLASS(self, op):
-        opinfo = self.getptrinfo(op.getarg(0))
+        opinfo = getptrinfo(op.getarg(0))
         expectedclassbox = op.getarg(1)
         assert isinstance(expectedclassbox, Const)
         if opinfo is not None:
@@ -563,8 +561,8 @@
                 # it was a guard_nonnull, which we replace with a
                 # guard_nonnull_class.
                 descr = compile.ResumeGuardDescr()
-                op = old_guard_op.copy_and_change (rop.GUARD_NONNULL_CLASS,
-                            args = [old_guard_op.getarg(0), op.getarg(1)],
+                op = old_guard_op.copy_and_change(rop.GUARD_NONNULL_CLASS,
+                            args=[old_guard_op.getarg(0), op.getarg(1)],
                             descr=descr)
                 # Note: we give explicitly a new descr for 'op'; this is why the
                 # old descr must not be ResumeAtPositionDescr (checked above).
@@ -577,14 +575,14 @@
 
     def postprocess_GUARD_CLASS(self, op):
         expectedclassbox = op.getarg(1)
-        info = self.getptrinfo(op.getarg(0))
+        info = getptrinfo(op.getarg(0))
         old_guard_op = info.get_last_guard(self.optimizer)
         update_last_guard = not old_guard_op or isinstance(
             old_guard_op.getdescr(), compile.ResumeAtPositionDescr)
         self.make_constant_class(op.getarg(0), expectedclassbox, update_last_guard)
 
     def optimize_GUARD_NONNULL_CLASS(self, op):
-        info = self.getptrinfo(op.getarg(0))
+        info = getptrinfo(op.getarg(0))
         if info and info.is_null():
             r = self.optimizer.metainterp_sd.logger_ops.repr_of_resop(op)
             raise InvalidLoop('A GUARD_NONNULL_CLASS (%s) was proven to '
@@ -663,8 +661,8 @@
     def _optimize_oois_ooisnot(self, op, expect_isnot, instance):
         arg0 = get_box_replacement(op.getarg(0))
         arg1 = get_box_replacement(op.getarg(1))
-        info0 = self.getptrinfo(arg0)
-        info1 = self.getptrinfo(arg1)
+        info0 = getptrinfo(arg0)
+        info1 = getptrinfo(arg1)
         if info0 and info0.is_virtual():
             if info1 and info1.is_virtual():
                 intres = (info0 is info1) ^ expect_isnot
@@ -724,8 +722,8 @@
         if length and length.getint() == 0:
             return None  # 0-length arraycopy
 
-        source_info = self.getptrinfo(op.getarg(1))
-        dest_info = self.getptrinfo(op.getarg(2))
+        source_info = getptrinfo(op.getarg(1))
+        dest_info = getptrinfo(op.getarg(2))
         source_start_box = self.get_constant_box(op.getarg(3))
         dest_start_box = self.get_constant_box(op.getarg(4))
         extrainfo = op.getdescr().get_extra_info()
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
@@ -25,9 +25,6 @@
     class cpu:
         remove_gctypeptr = True
 
-    def getptrinfo(self, box):
-        return box.get_forwarded()
-
     def setinfo_from_preamble(self, *args):
         pass
 
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -6,7 +6,7 @@
 from rpython.jit.metainterp.optimizeopt.util import (
     make_dispatcher_method, get_box_replacement)
 from rpython.jit.metainterp.optimizeopt.rawbuffer import InvalidRawOperation
-from .info import getrawptrinfo
+from .info import getrawptrinfo, getptrinfo
 from rpython.jit.metainterp.resoperation import rop, ResOperation
 
 
@@ -101,7 +101,7 @@
         effectinfo = op.getdescr().get_extra_info()
         oopspecindex = effectinfo.oopspecindex
         if oopspecindex == EffectInfo.OS_JIT_FORCE_VIRTUALIZABLE:
-            opinfo = self.getptrinfo(op.getarg(2))
+            opinfo = getptrinfo(op.getarg(2))
             if opinfo and opinfo.is_virtual():
                 return
         return self.emit(op)
@@ -161,17 +161,17 @@
         # was already forced).
 
     def _optimize_JIT_FORCE_VIRTUAL(self, op):
-        vref = self.getptrinfo(op.getarg(1))
+        vref = getptrinfo(op.getarg(1))
         vrefinfo = self.optimizer.metainterp_sd.virtualref_info
         if vref and vref.is_virtual():
             tokenop = vref.getfield(vrefinfo.descr_virtual_token, None)
             if tokenop is None:
                 return False
-            tokeninfo = self.getptrinfo(tokenop)
+            tokeninfo = getptrinfo(tokenop)
             if (tokeninfo is not None and tokeninfo.is_constant() and
                     not tokeninfo.is_nonnull()):
                 forcedop = vref.getfield(vrefinfo.descr_forced, None)
-                forcedinfo = self.getptrinfo(forcedop)
+                forcedinfo = getptrinfo(forcedop)
                 if forcedinfo is not None and not forcedinfo.is_null():
                     self.make_equal_to(op, forcedop)
                     self.last_emitted_operation = REMOVED
@@ -179,7 +179,7 @@
         return False
 
     def optimize_GETFIELD_GC_I(self, op):
-        opinfo = self.getptrinfo(op.getarg(0))
+        opinfo = getptrinfo(op.getarg(0))
         if opinfo and opinfo.is_virtual():
             fieldop = opinfo.getfield(op.getdescr())
             if fieldop is None:
@@ -193,7 +193,7 @@
 
     def optimize_SETFIELD_GC(self, op):
         struct = op.getarg(0)
-        opinfo = self.getptrinfo(struct)
+        opinfo = getptrinfo(struct)
         if opinfo is not None and opinfo.is_virtual():
             opinfo.setfield(op.getdescr(), struct,
                             get_box_replacement(op.getarg(1)))
@@ -230,7 +230,7 @@
             return self.do_RAW_FREE(op)
         elif effectinfo.oopspecindex == EffectInfo.OS_JIT_FORCE_VIRTUALIZABLE:
             # we might end up having CALL here instead of COND_CALL
-            info = self.getptrinfo(op.getarg(1))
+            info = getptrinfo(op.getarg(1))
             if info and info.is_virtual():
                 return
         else:
@@ -265,7 +265,7 @@
         return self.emit(op)
 
     def optimize_ARRAYLEN_GC(self, op):
-        opinfo = self.getptrinfo(op.getarg(0))
+        opinfo = getptrinfo(op.getarg(0))
         if opinfo and opinfo.is_virtual():
             self.make_constant_int(op, opinfo.getlength())
         else:
@@ -273,7 +273,7 @@
             return self.emit(op)
 
     def optimize_GETARRAYITEM_GC_I(self, op):
-        opinfo = self.getptrinfo(op.getarg(0))
+        opinfo = getptrinfo(op.getarg(0))
         if opinfo and opinfo.is_virtual():
             indexbox = self.get_constant_box(op.getarg(1))
             if indexbox is not None:
@@ -295,7 +295,7 @@
     optimize_GETARRAYITEM_GC_PURE_F = optimize_GETARRAYITEM_GC_I
 
     def optimize_SETARRAYITEM_GC(self, op):
-        opinfo = self.getptrinfo(op.getarg(0))
+        opinfo = getptrinfo(op.getarg(0))
         if opinfo and opinfo.is_virtual():
             indexbox = self.get_constant_box(op.getarg(1))
             if indexbox is not None:
@@ -384,7 +384,7 @@
         return self.emit(op)
 
     def optimize_GETINTERIORFIELD_GC_I(self, op):
-        opinfo = self.getptrinfo(op.getarg(0))
+        opinfo = getptrinfo(op.getarg(0))
         if opinfo and opinfo.is_virtual():
             indexbox = self.get_constant_box(op.getarg(1))
             if indexbox is not None:
@@ -402,7 +402,7 @@
     optimize_GETINTERIORFIELD_GC_F = optimize_GETINTERIORFIELD_GC_I
 
     def optimize_SETINTERIORFIELD_GC(self, op):
-        opinfo = self.getptrinfo(op.getarg(0))
+        opinfo = getptrinfo(op.getarg(0))
         if opinfo and opinfo.is_virtual():
             indexbox = self.get_constant_box(op.getarg(1))
             if indexbox is not None:
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -6,6 +6,7 @@
      MININT, MAXINT, IntBound, IntLowerBound
 from rpython.jit.metainterp.resoperation import rop, ResOperation, \
      InputArgInt, InputArgRef, InputArgFloat
+from .info import getptrinfo
 from .util import get_box_replacement
 from rpython.rlib.debug import debug_print
 
@@ -145,7 +146,7 @@
         assert len(self.fielddescrs) == len(self.fieldstate)
         assert len(other.fielddescrs) == len(other.fieldstate)
         if runtime_box is not None:
-            opinfo = state.optimizer.getptrinfo(box)
+            opinfo = getptrinfo(box)
             assert opinfo.is_virtual()
             assert isinstance(opinfo, AbstractStructPtrInfo)
         else:
@@ -180,7 +181,7 @@
 
     def enum_forced_boxes(self, boxes, box, optimizer, force_boxes=False):
         box = get_box_replacement(box)
-        info = optimizer.getptrinfo(box)
+        info = getptrinfo(box)
         if info is None or not info.is_virtual():
             raise VirtualStatesCantMatch()
         else:
@@ -248,7 +249,7 @@
         fieldbox_runtime = None
         for i in range(len(self.fieldstate)):
             if runtime_box is not None:
-                opinfo = state.optimizer.getptrinfo(box)
+                opinfo = getptrinfo(box)
                 assert isinstance(opinfo, ArrayPtrInfo)
                 fieldbox = opinfo._items[i]
                 fieldbox_runtime = state.get_runtime_item(runtime_box,
@@ -261,7 +262,7 @@
 
     def enum_forced_boxes(self, boxes, box, optimizer, force_boxes=False):
         box = get_box_replacement(box)
-        info = optimizer.getptrinfo(box)
+        info = getptrinfo(box)
         if info is None or not info.is_virtual():
             raise VirtualStatesCantMatch()
         if len(self.fieldstate) > info.getlength():
@@ -305,7 +306,7 @@
         fieldbox = None
         fieldbox_runtime = None
         if box is not None:
-            opinfo = state.optimizer.getptrinfo(box)
+            opinfo = getptrinfo(box)
             assert isinstance(opinfo, ArrayPtrInfo)
         else:
             opinfo = None
@@ -330,7 +331,7 @@
                 s.enum(virtual_state)
 
     def enum_forced_boxes(self, boxes, box, optimizer, force_boxes=False):
-        opinfo = optimizer.getptrinfo(box)
+        opinfo = getptrinfo(box)
         if not isinstance(opinfo, ArrayStructInfo):
             raise VirtualStatesCantMatch
         if not opinfo.is_virtual():
@@ -412,15 +413,15 @@
         if self.level == LEVEL_CONSTANT:
             return
         assert 0 <= self.position_in_notvirtuals
-        if optimizer is not None:
-            box = get_box_replacement(box)
-            if box.type == 'r':
-                info = optimizer.getptrinfo(box)
-                if info and info.is_virtual():
-                    if force_boxes:
-                        info.force_box(box, optimizer)
-                    else:
-                        raise VirtualStatesCantMatch
+        assert optimizer is not None
+        box = get_box_replacement(box)
+        if box.type == 'r':
+            info = getptrinfo(box)
+            if info and info.is_virtual():
+                if force_boxes:
+                    info.force_box(box, optimizer)
+                else:
+                    raise VirtualStatesCantMatch
         boxes[self.position_in_notvirtuals] = box
 
     def _enum(self, virtual_state):
@@ -714,11 +715,11 @@
         except KeyError:
             pass
         if box.type == 'r':
-            info = opt.getptrinfo(box)
+            info = getptrinfo(box)
             if info is not None and info.is_virtual():
                 result = info.visitor_dispatch_virtual_type(self)
                 self.info[box] = result
-                info.visitor_walk_recursive(box, self, opt)
+                info.visitor_walk_recursive(box, self)
                 result.fieldstate = [self.create_state_or_none(b, opt)
                                      for b in self.fieldboxes[box]]
             else:
diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py
--- a/rpython/jit/metainterp/optimizeopt/vstring.py
+++ b/rpython/jit/metainterp/optimizeopt/vstring.py
@@ -7,7 +7,7 @@
 from rpython.jit.metainterp.optimizeopt.util import (
     make_dispatcher_method, get_box_replacement)
 from rpython.jit.metainterp.resoperation import rop, ResOperation
-from rpython.jit.metainterp.optimizeopt import info
+from .info import AbstractVirtualPtrInfo, getptrinfo
 from rpython.rlib.objectmodel import specialize, we_are_translated
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rtyper import annlowlevel
@@ -47,7 +47,7 @@
 # ____________________________________________________________
 
 
-class StrPtrInfo(info.AbstractVirtualPtrInfo):
+class StrPtrInfo(AbstractVirtualPtrInfo):
     #_attrs_ = ('length', 'lenbound', 'lgtop', 'mode', '_cached_vinfo', '_is_virtual')
 
     lenbound = None
@@ -119,7 +119,7 @@
         return lengthop
 
     def make_guards(self, op, short, optimizer):
-        info.AbstractVirtualPtrInfo.make_guards(self, op, short, optimizer)
+        AbstractVirtualPtrInfo.make_guards(self, op, short, optimizer)
         if (self.lenbound and
                 self.lenbound.has_lower and self.lenbound.lower >= 1):
             if self.mode is mode_string:
@@ -205,7 +205,7 @@
             offsetbox = _int_add(optstring, offsetbox, CONST_1)
         return offsetbox
 
-    def _visitor_walk_recursive(self, instbox, visitor, optimizer):
+    def _visitor_walk_recursive(self, instbox, visitor):
         visitor.register_virtual_fields(instbox, self._chars)
 
     @specialize.argtype(1)
@@ -238,7 +238,7 @@
         vstart = optstring.getintbound(self.start)
         vlength = optstring.getintbound(self.lgtop)
         if vstart.is_constant() and vlength.is_constant():
-            vstr = optstring.getptrinfo(self.s)
+            vstr = getptrinfo(self.s)
             s1 = vstr.get_constant_string_spec(optstring, mode)
             if s1 is None:
                 return None
@@ -253,12 +253,12 @@
         assert op is not None
         return self.lgtop
 
-    def _visitor_walk_recursive(self, instbox, visitor, optimizer):
+    def _visitor_walk_recursive(self, instbox, visitor):
         boxes = [self.s, self.start, self.lgtop]
         visitor.register_virtual_fields(instbox, boxes)
-        opinfo = optimizer.getptrinfo(self.s)
+        opinfo = getptrinfo(self.s)
         if opinfo and opinfo.is_virtual():
-            opinfo.visitor_walk_recursive(self.s, visitor, optimizer)
+            opinfo.visitor_walk_recursive(self.s, visitor)
 
     @specialize.argtype(1)
     def visitor_dispatch_virtual_type(self, visitor):
@@ -283,11 +283,11 @@
         assert op is not None
         if self.lgtop is not None:
             return self.lgtop
-        lefti = optstring.getptrinfo(self.vleft)
+        lefti = getptrinfo(self.vleft)
         len1box = lefti.getstrlen(self.vleft, optstring, mode)
         if len1box is None:
             return None
-        righti = optstring.getptrinfo(self.vright)
+        righti = getptrinfo(self.vright)
         len2box = righti.getstrlen(self.vright, optstring, mode)
         if len2box is None:
             return None
@@ -297,11 +297,11 @@
 
     @specialize.arg(2)
     def get_constant_string_spec(self, optstring, mode):
-        ileft = optstring.getptrinfo(self.vleft)
+        ileft = getptrinfo(self.vleft)
         s1 = ileft.get_constant_string_spec(optstring, mode)
         if s1 is None:
             return None
-        iright = optstring.getptrinfo(self.vright)
+        iright = getptrinfo(self.vright)
         s2 = iright.get_constant_string_spec(optstring, mode)
         if s2 is None:
             return None
@@ -309,26 +309,26 @@
 
     def string_copy_parts(self, op, optstring, targetbox, offsetbox,
                           mode):
-        lefti = optstring.getptrinfo(self.vleft)
+        lefti = getptrinfo(self.vleft)
         offsetbox = lefti.string_copy_parts(self.vleft, optstring,
                                             targetbox, offsetbox, mode)
-        righti = optstring.getptrinfo(self.vright)
+        righti = getptrinfo(self.vright)
         offsetbox = righti.string_copy_parts(self.vright, optstring,
                                              targetbox, offsetbox, mode)
         return offsetbox
 
-    def _visitor_walk_recursive(self, instbox, visitor, optimizer):
+    def _visitor_walk_recursive(self, instbox, visitor):
         # we don't store the lengthvalue in guards, because the
         # guard-failed code starts with a regular STR_CONCAT again
         leftbox = self.vleft
         rightbox = self.vright
         visitor.register_virtual_fields(instbox, [leftbox, rightbox])
-        leftinfo = optimizer.getptrinfo(leftbox)
-        rightinfo = optimizer.getptrinfo(rightbox)
+        leftinfo = getptrinfo(leftbox)
+        rightinfo = getptrinfo(rightbox)
         if leftinfo and leftinfo.is_virtual():
-            leftinfo.visitor_walk_recursive(leftbox, visitor, optimizer)
+            leftinfo.visitor_walk_recursive(leftbox, visitor)
         if rightinfo and rightinfo.is_virtual():
-            rightinfo.visitor_walk_recursive(rightbox, visitor, optimizer)
+            rightinfo.visitor_walk_recursive(rightbox, visitor)
 
     @specialize.argtype(1)
     def visitor_dispatch_virtual_type(self, visitor):
@@ -463,7 +463,7 @@
         self.pure_from_args(mode_unicode.STRLEN, [op], op.getarg(0))
 
     def optimize_STRSETITEM(self, op):
-        opinfo = self.getptrinfo(op.getarg(0))
+        opinfo = getptrinfo(op.getarg(0))
         if opinfo:
             assert not opinfo.is_constant()
             # strsetitem(ConstPtr) never makes sense
@@ -489,12 +489,12 @@
 
     def strgetitem(self, op, s, index, mode):
         self.make_nonnull_str(s, mode)
-        sinfo = self.getptrinfo(s)
+        sinfo = getptrinfo(s)
         #
         if isinstance(sinfo, VStringSliceInfo) and sinfo.is_virtual(): # slice
             index = _int_add(self.optimizer, sinfo.start, index)
             s = sinfo.s
-            sinfo = self.getptrinfo(sinfo.s)
+            sinfo = getptrinfo(sinfo.s)
         #
         if isinstance(sinfo, VStringPlainInfo):
             # even if no longer virtual
@@ -508,7 +508,7 @@
         #
         vindex = self.getintbound(index)
         if isinstance(sinfo, VStringConcatInfo) and vindex.is_constant():
-            leftinfo = self.getptrinfo(sinfo.vleft)
+            leftinfo = getptrinfo(sinfo.vleft)
             len1box = leftinfo.getstrlen(sinfo.vleft, self, mode)
             if isinstance(len1box, ConstInt):
                 raw_index = vindex.getint()
@@ -529,7 +529,7 @@
 
     def _optimize_STRLEN(self, op, mode):
         arg1 = get_box_replacement(op.getarg(0))
-        opinfo = self.getptrinfo(arg1)
+        opinfo = getptrinfo(arg1)
         if opinfo:
             lgtop = opinfo.getstrlen(arg1, self, mode)
             if lgtop is not None:
@@ -544,7 +544,7 @@
         return self._optimize_STRHASH(op, mode_unicode)
 
     def _optimize_STRHASH(self, op, mode):
-        opinfo = self.getptrinfo(op.getarg(0))
+        opinfo = getptrinfo(op.getarg(0))
         if opinfo:
             lgtop = opinfo.getstrhash(op, mode)
             if lgtop is not None:
@@ -565,8 +565,8 @@
         assert op.getarg(2).type == INT
         assert op.getarg(3).type == INT
         assert op.getarg(4).type == INT
-        src = self.getptrinfo(op.getarg(0))
-        dst = self.getptrinfo(op.getarg(1))
+        src = getptrinfo(op.getarg(0))
+        dst = getptrinfo(op.getarg(1))
         srcstart = self.getintbound(op.getarg(2))
         dststart = self.getintbound(op.getarg(3))
         length = self.getintbound(op.getarg(4))
@@ -641,7 +641,7 @@
         # More generally, supporting non-constant but virtual cases is
         # not obvious, because of the exception UnicodeDecodeError that
         # can be raised by ll_str2unicode()
-        varg = self.getptrinfo(op.getarg(1))
+        varg = getptrinfo(op.getarg(1))
         s = None
         if varg:
             s = varg.get_constant_string_spec(self, mode_string)
@@ -666,7 +666,7 @@
 
     def opt_call_stroruni_STR_SLICE(self, op, mode):
         self.make_nonnull_str(op.getarg(1), mode)
-        vstr = self.getptrinfo(op.getarg(1))
+        vstr = getptrinfo(op.getarg(1))
         vstart = self.getintbound(op.getarg(2))
         vstop = self.getintbound(op.getarg(3))
         #
@@ -697,8 +697,8 @@
     def opt_call_stroruni_STR_EQUAL(self, op, mode):
         arg1 = get_box_replacement(op.getarg(1))
         arg2 = get_box_replacement(op.getarg(2))
-        i1 = self.getptrinfo(arg1)
-        i2 = self.getptrinfo(arg2)
+        i1 = getptrinfo(arg1)
+        i2 = getptrinfo(arg2)
         #
         if i1:
             l1box = i1.getstrlen(arg1, self, mode)
@@ -738,8 +738,8 @@
         return False, None
 
     def handle_str_equal_level1(self, arg1, arg2, resultop, mode):
-        i1 = self.getptrinfo(arg1)
-        i2 = self.getptrinfo(arg2)
+        i1 = getptrinfo(arg1)
+        i2 = getptrinfo(arg2)
         l2box = None
         l1box = None
         if i2:
@@ -748,7 +748,7 @@
             if l2box.value == 0:
                 if i1 and i1.is_nonnull():
                     self.make_nonnull_str(arg1, mode)
-                    i1 = self.getptrinfo(arg1)
+                    i1 = getptrinfo(arg1)
                     lengthbox = i1.getstrlen(arg1, self, mode)
                 else:
                     lengthbox = None
@@ -792,8 +792,8 @@
         return False, None
 
     def handle_str_equal_level2(self, arg1, arg2, resultbox, mode):
-        i1 = self.getptrinfo(arg1)
-        i2 = self.getptrinfo(arg2)
+        i1 = getptrinfo(arg1)
+        i2 = getptrinfo(arg2)
         l2box = None
         if i2:
             l2box = i2.getstrlen(arg1, self, mode)
@@ -821,8 +821,8 @@
     def opt_call_stroruni_STR_CMP(self, op, mode):
         arg1 = get_box_replacement(op.getarg(1))
         arg2 = get_box_replacement(op.getarg(2))
-        i1 = self.getptrinfo(arg1)
-        i2 = self.getptrinfo(arg2)
+        i1 = getptrinfo(arg1)
+        i2 = getptrinfo(arg2)
         if not i1 or not i2:
             return False, None
         l1box = i1.getstrlen(arg1, self, mode)
@@ -842,7 +842,7 @@
         return False, None
 
     def opt_call_SHRINK_ARRAY(self, op):
-        i1 = self.getptrinfo(op.getarg(1))
+        i1 = getptrinfo(op.getarg(1))
         i2 = self.getintbound(op.getarg(2))
         # If the index is constant, if the argument is virtual (we only support
         # VStringPlainValue for now) we can optimize away the call.
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -221,10 +221,11 @@
 
     # env numbering
 
-    def _number_boxes(self, iter, arr, optimizer, numb_state):
+    def _number_boxes(self, iter, arr, numb_state):
         """ Number boxes from one snapshot
         """
-        from rpython.jit.metainterp.optimizeopt.info import getrawptrinfo
+        from rpython.jit.metainterp.optimizeopt.info import (
+            getrawptrinfo, getptrinfo)
         num_boxes = numb_state.num_boxes
         num_virtuals = numb_state.num_virtuals
         liveboxes = numb_state.liveboxes
@@ -239,7 +240,7 @@
             else:
                 is_virtual = False
                 if box.type == 'r':
-                    info = optimizer.getptrinfo(box)
+                    info = getptrinfo(box)
                     is_virtual = (info is not None and info.is_virtual())
                 if box.type == 'i':
                     info = getrawptrinfo(box)
@@ -264,21 +265,20 @@
         arr = snapshot_iter.vable_array
 
         numb_state.append_int(len(arr))
-        self._number_boxes(snapshot_iter, arr, optimizer, numb_state)
+        self._number_boxes(snapshot_iter, arr, numb_state)
 
         arr = snapshot_iter.vref_array
         n = len(arr)
         assert not (n & 1)
         numb_state.append_int(n >> 1)
 
-        self._number_boxes(snapshot_iter, arr, optimizer, numb_state)
+        self._number_boxes(snapshot_iter, arr, numb_state)
 
         for snapshot in snapshot_iter.framestack:
             jitcode_index, pc = snapshot_iter.unpack_jitcode_pc(snapshot)
             numb_state.append_int(jitcode_index)
             numb_state.append_int(pc)
-            self._number_boxes(
-                    snapshot_iter, snapshot.box_array, optimizer, numb_state)
+            self._number_boxes(snapshot_iter, snapshot.box_array, numb_state)
         numb_state.patch_current_size(0)
 
         return numb_state
@@ -415,7 +415,8 @@
         return tagbits == TAGVIRTUAL
 
     def finish(self, pending_setfields=[]):
-        from rpython.jit.metainterp.optimizeopt.info import getrawptrinfo
+        from rpython.jit.metainterp.optimizeopt.info import (
+            getrawptrinfo, getptrinfo)
         optimizer = self.optimizer
         # compute the numbering
         storage = self.storage
@@ -441,12 +442,12 @@
             else:
                 assert tagbits == TAGVIRTUAL
                 if box.type == 'r':
-                    info = optimizer.getptrinfo(box)
+                    info = getptrinfo(box)
                 else:
                     assert box.type == 'i'
                     info = getrawptrinfo(box)
                 assert info.is_virtual()
-                info.visitor_walk_recursive(box, self, optimizer)
+                info.visitor_walk_recursive(box, self)
 
         for setfield_op in pending_setfields:
             box = setfield_op.getarg(0)
@@ -458,11 +459,11 @@
             fieldbox = optimizer.get_box_replacement(fieldbox)
             self.register_box(box)
             self.register_box(fieldbox)
-            info = optimizer.getptrinfo(fieldbox)
+            info = getptrinfo(fieldbox)
             assert info is not None and info.is_virtual()
-            info.visitor_walk_recursive(fieldbox, self, optimizer)
+            info.visitor_walk_recursive(fieldbox, self)
 
-        self._number_virtuals(liveboxes, optimizer, num_virtuals)
+        self._number_virtuals(liveboxes, num_virtuals)
         self._add_pending_fields(optimizer, pending_setfields)
 
         numb_state.patch(1, len(liveboxes))
@@ -472,8 +473,9 @@
         storage.rd_consts = self.memo.consts
         return liveboxes[:]
 
-    def _number_virtuals(self, liveboxes, optimizer, num_env_virtuals):
-        from rpython.jit.metainterp.optimizeopt.info import AbstractVirtualPtrInfo
+    def _number_virtuals(self, liveboxes, num_env_virtuals):
+        from rpython.jit.metainterp.optimizeopt.info import (
+            AbstractVirtualPtrInfo, getptrinfo)
 
         # !! 'liveboxes' is a list that is extend()ed in-place !!
         memo = self.memo
@@ -514,7 +516,7 @@
             memo.nvholes += length - len(vfieldboxes)
             for virtualbox, fieldboxes in vfieldboxes.iteritems():
                 num, _ = untag(self.liveboxes[virtualbox])
-                info = optimizer.getptrinfo(virtualbox)
+                info = getptrinfo(virtualbox)
                 assert info.is_virtual()
                 assert isinstance(info, AbstractVirtualPtrInfo)
                 fieldnums = [self._gettagged(box)
diff --git a/rpython/jit/metainterp/test/test_bridgeopt.py b/rpython/jit/metainterp/test/test_bridgeopt.py
--- a/rpython/jit/metainterp/test/test_bridgeopt.py
+++ b/rpython/jit/metainterp/test/test_bridgeopt.py
@@ -28,14 +28,10 @@
     optheap = None
     optrewrite = None
 
-    def __init__(self, dct={}, cpu=None):
-        self.dct = dct
+    def __init__(self, cpu=None):
         self.constant_classes = {}
         self.cpu = cpu
 
-    def getptrinfo(self, arg):
-        return self.dct.get(arg, None)
-
     def make_constant_class(self, arg, cls):
         self.constant_classes[arg] = cls
 
@@ -47,13 +43,12 @@
         self.rd_numb = numb
 
 def test_known_classes():
+    cls = FakeClass()
     box1 = InputArgRef()
+    box1.set_forwarded(InstancePtrInfo(known_class=cls))
     box2 = InputArgRef()
     box3 = InputArgRef()
-
-    cls = FakeClass()
-    dct = {box1: InstancePtrInfo(known_class=cls)}
-    optimizer = FakeOptimizer(dct)
+    optimizer = FakeOptimizer()
 
     numb_state = NumberingState(4)
     numb_state.append_int(1) # size of resume block
@@ -87,10 +82,10 @@
 @given(boxes_known_classes)
 def test_random_class_knowledge(boxes_known_classes):
     cls = FakeClass()
-    dct1 = {box: InstancePtrInfo(known_class=cls)
-              for box, known_class in boxes_known_classes
-                  if known_class}
-    optimizer = FakeOptimizer(dct1)
+    for box, known_class in boxes_known_classes:
+        if known_class:
+            box.set_forwarded(InstancePtrInfo(known_class=cls))
+    optimizer = FakeOptimizer()
 
     refboxes = [box for (box, _) in boxes_known_classes
                     if isinstance(box, InputArgRef)]
diff --git a/rpython/jit/metainterp/test/test_resume.py b/rpython/jit/metainterp/test/test_resume.py
--- a/rpython/jit/metainterp/test/test_resume.py
+++ b/rpython/jit/metainterp/test/test_resume.py
@@ -1,4 +1,3 @@
-from __future__ import with_statement
 import py
 import sys
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
@@ -15,7 +14,7 @@
     unpack_numbering, create_numbering)
 from rpython.jit.metainterp.opencoder import Trace
 
-from rpython.jit.metainterp.optimizeopt import info, util
+from rpython.jit.metainterp.optimizeopt import info
 from rpython.jit.metainterp.history import (
     ConstInt, Const, AbstractDescr, ConstPtr, ConstFloat, IntFrontendOp,
     RefFrontendOp, CONST_NULL)
@@ -51,10 +50,6 @@
             op = op.get_forwarded()
         return op
 
-    def getptrinfo(self, op):
-        op = util.get_box_replacement(op)
-        return op.get_forwarded()
-
 
 # ____________________________________________________________
 
@@ -872,12 +867,12 @@
                                   False, [], env3)
     snap3.prev = snap
 
-    class FakeVirtualInfo(info.AbstractInfo):
+    class FakeVirtualInfo(info.AbstractVirtualPtrInfo):
         def __init__(self, virt):
-            self.virt = virt
+            self._is_virtual = virt
 
         def is_virtual(self):
-            return self.virt
+            return self._is_virtual
 
     # renamed
     b3.set_forwarded(c4)
@@ -1195,7 +1190,7 @@
 
 
 def test_virtual_adder_make_virtual():
-    b2s, b3s, b4s, b5s = [IntFrontendOp(0), IntFrontendOp(0), RefFrontendOp(0),
+    b2s, b3s, b4s, b5s = [RefFrontendOp(0), IntFrontendOp(0), RefFrontendOp(0),
                           RefFrontendOp(0)]
     c1s = ConstInt(111)
     storage = Storage()
@@ -1222,7 +1217,7 @@
     modifier.register_virtual_fields(b4s, [b3s, None, None, None, b2s, b5s])
 
     liveboxes = []
-    modifier._number_virtuals(liveboxes, FakeOptimizer(), 0)
+    modifier._number_virtuals(liveboxes, 0)
     storage.rd_consts = memo.consts[:]
     storage.rd_numb = Numbering([0])
     # resume
@@ -1277,7 +1272,7 @@
         del Const.__eq__
 
 def test_virtual_adder_make_varray():
-    b2s, b4s = [IntFrontendOp(0), IntFrontendOp(0)]
+    b2s, b4s = [RefFrontendOp(0), IntFrontendOp(0)]
     b4s.setint(4)
     c1s = ConstInt(111)
     storage = Storage()
@@ -1292,7 +1287,7 @@
     v2._items = [b4s, c1s]
     modifier.register_virtual_fields(b2s, [b4s, c1s])
     liveboxes = []
-    modifier._number_virtuals(liveboxes, FakeOptimizer(), 0)
+    modifier._number_virtuals(liveboxes, 0)
     dump_storage(storage, liveboxes)
     storage.rd_consts = memo.consts[:]
     storage.rd_numb = Numbering([0])
@@ -1345,7 +1340,7 @@
     v2.setfield(LLtypeMixin.bdescr, b2s, b4s)
     modifier.register_virtual_fields(b2s, [c1s, c1s, b4s])
     liveboxes = []
-    modifier._number_virtuals(liveboxes, FakeOptimizer(), 0)
+    modifier._number_virtuals(liveboxes, 0)
     dump_storage(storage, liveboxes)
     storage.rd_consts = memo.consts[:]
     storage.rd_numb = Numbering([0])
@@ -1389,7 +1384,7 @@
     modifier.register_box(b4s)
 
     liveboxes = []
-    modifier._number_virtuals(liveboxes, FakeOptimizer(), 0)
+    modifier._number_virtuals(liveboxes, 0)
     assert liveboxes == [b2s, b4s] or liveboxes == [b4s, b2s]
     modifier._add_pending_fields(FakeOptimizer(), [
         ResOperation(rop.SETFIELD_GC, [b2s, b4s], descr=LLtypeMixin.nextdescr)])

From pypy.commits at gmail.com  Wed May 15 11:53:54 2019
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 15 May 2019 08:53:54 -0700 (PDT)
Subject: [pypy-commit] pypy optimizeopt-cleanup: simplify metainterp_sd
 creation in optimizeopt tests
Message-ID: <5cdc3612.1c69fb81.5b966.f7cd@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: optimizeopt-cleanup
Changeset: r96622:0aaebc5466a7
Date: 2019-05-15 16:52 +0100
http://bitbucket.org/pypy/pypy/changeset/0aaebc5466a7/

Log:	simplify metainterp_sd creation in optimizeopt tests

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py b/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py
@@ -7,7 +7,7 @@
 from rpython.jit.metainterp.optimizeopt.dependency import DependencyGraph
 from rpython.jit.metainterp.optimizeopt.test.test_schedule import SchedulerBaseTest
 from rpython.jit.metainterp.optimizeopt.test.test_vecopt import (
-    FakeMetaInterpStaticData, FakeJitDriverStaticData)
+    FakeJitDriverStaticData)
 
 class FakeMemoryRef(object):
     def __init__(self, array, iv):
@@ -80,9 +80,8 @@
 
 class TestCostModel(SchedulerBaseTest):
     def savings(self, loop):
-        metainterp_sd = FakeMetaInterpStaticData(self.cpu)
         jitdriver_sd = FakeJitDriverStaticData()
-        opt = VectorizingOptimizer(metainterp_sd, jitdriver_sd, 0)
+        opt = VectorizingOptimizer(self.metainterp_sd, jitdriver_sd, 0)
         opt.orig_label_args = loop.label.getarglist()[:]
         graph = opt.dependency_graph = DependencyGraph(loop)
         self.show_dot_graph(graph, 'costmodel')
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_dependency.py b/rpython/jit/metainterp/optimizeopt/test/test_dependency.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_dependency.py
@@ -7,8 +7,7 @@
         IndexVar, MemoryRef, Node)
 from rpython.jit.metainterp.optimizeopt.vector import VectorLoop
 from rpython.jit.metainterp.optimizeopt.test.test_util import (
-    BaseTest, FakeMetaInterpStaticData, convert_old_style_to_targets,
-    FakeJitDriverStaticData)
+    BaseTest, convert_old_style_to_targets, FakeJitDriverStaticData)
 from rpython.jit.metainterp.resoperation import rop, ResOperation
 from rpython.jit.backend.llgraph.runner import ArrayDescr
 from rpython.jit.tool.oparser import OpParser
@@ -109,7 +108,7 @@
         loop = VectorLoop(label, loop.operations[0:-1], jump)
         loop.jump.setdescr(token)
         class Optimizer(object):
-            metainterp_sd = FakeMetaInterpStaticData(self.cpu)
+            metainterp_sd = self.metainterp_sd
             jitdriver_sd = FakeJitDriverStaticData()
         opt = Optimizer()
         opt.jitdriver_sd.vec = True
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -6,7 +6,7 @@
 from rpython.rtyper import rclass
 from rpython.rtyper.lltypesystem import lltype
 from rpython.jit.metainterp.optimizeopt.test.test_util import (
-    BaseTest, FakeMetaInterpStaticData, convert_old_style_to_targets)
+    BaseTest, convert_old_style_to_targets)
 from rpython.jit.metainterp.history import (
     JitCellToken, ConstInt, get_const_ptr_for_string)
 from rpython.jit.metainterp import executor, compile
@@ -32,7 +32,7 @@
         exp = parse(optops, namespace=self.namespace.copy())
         expected = convert_old_style_to_targets(exp, jump=True)
         call_pure_results = self._convert_call_pure_results(call_pure_results)
-        trace = convert_loop_to_trace(loop, FakeMetaInterpStaticData(self.cpu))
+        trace = convert_loop_to_trace(loop, self.metainterp_sd)
         compile_data = compile.SimpleCompileData(
             trace, call_pure_results=call_pure_results)
         info, ops = self._do_optimize_loop(compile_data)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py
@@ -1,5 +1,5 @@
 from rpython.jit.metainterp.optimizeopt.test.test_util import (
-    BaseTest, convert_old_style_to_targets, FakeMetaInterpStaticData)
+    BaseTest, convert_old_style_to_targets)
 from rpython.jit.metainterp import compile
 from rpython.jit.tool import oparser
 from rpython.jit.metainterp.resoperation import ResOperation, rop
@@ -27,8 +27,7 @@
         bridge = self.parse(bridge_ops)
         bridge.operations[-1].setdescr(jitcell_token)
         self.add_guard_future_condition(bridge)
-        trace = oparser.convert_loop_to_trace(
-            bridge, FakeMetaInterpStaticData(self.cpu))
+        trace = oparser.convert_loop_to_trace(bridge, self.metainterp_sd)
         data = compile.BridgeCompileData(
             trace,
             self.convert_values(bridge.operations[-1].getarglist(), bridge_values),
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
@@ -13,7 +13,7 @@
 from rpython.jit.metainterp.optimizeopt.test.test_dependency import (
     DependencyBaseTest)
 from rpython.jit.metainterp.optimizeopt.test.test_vecopt import (
-    FakeMetaInterpStaticData, FakeJitDriverStaticData, FakePackSet)
+    FakeJitDriverStaticData, FakePackSet)
 from rpython.jit.metainterp.resoperation import (
     rop, ResOperation, VectorizationInfo)
 
@@ -60,9 +60,8 @@
         for name, overwrite in (overwrite_funcs or {}).items():
             setattr(state, name, overwrite)
         renamer = Renamer()
-        metainterp_sd = FakeMetaInterpStaticData(self.cpu)
         jitdriver_sd = FakeJitDriverStaticData()
-        opt = VectorizingOptimizer(metainterp_sd, jitdriver_sd, 0)
+        opt = VectorizingOptimizer(self.metainterp_sd, jitdriver_sd, 0)
         opt.packset = packset
         opt.combine_packset()
         opt.schedule(state)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
@@ -2,8 +2,7 @@
 More direct tests for unrolling
 """
 
-from rpython.jit.metainterp.optimizeopt.test.test_util import (
-    BaseTest, FakeMetaInterpStaticData)
+from rpython.jit.metainterp.optimizeopt.test.test_util import BaseTest
 from rpython.jit.metainterp.optimizeopt.util import equaloplists
 from rpython.jit.metainterp.history import TreeLoop
 from rpython.jit.metainterp.resoperation import (
@@ -42,7 +41,7 @@
         assert jumpop.getopnum() == rop.JUMP
         inputargs = loop.inputargs
         preamble = TreeLoop('preamble')
-        trace = oparser.convert_loop_to_trace(loop, FakeMetaInterpStaticData(self.cpu))
+        trace = oparser.convert_loop_to_trace(loop, self.metainterp_sd)
         compile_data = PreambleCompileData(trace, inputargs)
         start_state, newops = self._do_optimize_loop(compile_data)
         preamble.operations = newops
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -515,6 +515,13 @@
 
 
 class BaseTest(LLtypeMixin):
+    @pytest.fixture(autouse=True)
+    def cls_attributes(self):
+        metainterp_sd = FakeMetaInterpStaticData(self.cpu)
+        metainterp_sd.virtualref_info = self.vrefinfo
+        compute_bitstrings(self.cpu.fetch_all_descrs())
+        self.metainterp_sd = metainterp_sd
+
     def parse(self, s, boxkinds=None, want_fail_descr=True, postprocess=None):
         AbstractValue._repr_memo.counter = 0
         self.oparse = OpParser(s, self.cpu, self.namespace, boxkinds,
@@ -538,12 +545,8 @@
                             expected.operations, False, remap, text_right)
 
     def _do_optimize_loop(self, compile_data):
-        metainterp_sd = FakeMetaInterpStaticData(self.cpu)
-        metainterp_sd.virtualref_info = self.vrefinfo
-        compute_bitstrings(self.cpu.fetch_all_descrs())
-        #
         compile_data.enable_opts = self.enable_opts
-        state = compile_data.optimize_trace(metainterp_sd, None, {})
+        state = compile_data.optimize_trace(self.metainterp_sd, None, {})
         return state
 
     def _convert_call_pure_results(self, d):
@@ -580,7 +583,7 @@
         runtime_boxes = self.pack_into_boxes(jump_op, jump_values)
         jump_op.setdescr(celltoken)
         call_pure_results = self._convert_call_pure_results(call_pure_results)
-        t = convert_loop_to_trace(loop, FakeMetaInterpStaticData(self.cpu))
+        t = convert_loop_to_trace(loop, self.metainterp_sd)
         preamble_data = compile.PreambleCompileData(
             t, runtime_boxes, call_pure_results, enable_opts=self.enable_opts)
         start_state, preamble_ops = self._do_optimize_loop(preamble_data)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py b/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
@@ -2,8 +2,6 @@
 import sys
 import pytest
 
-from rpython.jit.metainterp.optimizeopt.test.test_util import (
-        FakeMetaInterpStaticData)
 from rpython.jit.metainterp.optimizeopt.test.test_dependency import DependencyBaseTest
 from rpython.jit.metainterp.optimizeopt.vector import (VectorizingOptimizer,
         MemoryRef, isomorphic, Pair, NotAVectorizeableLoop,
@@ -80,13 +78,12 @@
 
     def assert_vectorize(self, loop, expected_loop, call_pure_results=None):
         jump = ResOperation(rop.JUMP, loop.jump.getarglist(), loop.jump.getdescr())
-        metainterp_sd = FakeMetaInterpStaticData(self.cpu)
         warmstate = FakeWarmState()
         loop.operations += [loop.jump]
         loop_info = BasicLoopInfo(loop.jump.getarglist(), None, jump)
         loop_info.label_op = ResOperation(
             rop.LABEL, loop.jump.getarglist(), loop.jump.getdescr())
-        optimize_vector(None, metainterp_sd, self.jitdriver_sd, warmstate,
+        optimize_vector(None, self.metainterp_sd, self.jitdriver_sd, warmstate,
                         loop_info, loop.operations)
         loop.operations = loop.operations[:-1]
         #loop.label = state[0].label_op
@@ -94,9 +91,8 @@
         self.assert_equal(loop, expected_loop)
 
     def vectoroptimizer(self, loop):
-        metainterp_sd = FakeMetaInterpStaticData(self.cpu)
         jitdriver_sd = FakeJitDriverStaticData()
-        opt = VectorizingOptimizer(metainterp_sd, jitdriver_sd, 0)
+        opt = VectorizingOptimizer(self.metainterp_sd, jitdriver_sd, 0)
         opt.orig_label_args = loop.label.getarglist()[:]
         return opt
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
@@ -1,25 +1,23 @@
 from __future__ import with_statement
 import py
-from rpython.jit.metainterp.optimizeopt.virtualstate import VirtualStateInfo,\
-     VStructStateInfo, LEVEL_CONSTANT,\
-     VArrayStateInfo, not_virtual, VirtualState,\
-     GenerateGuardState, VirtualStatesCantMatch, VArrayStructStateInfo,\
-     VirtualStateConstructor
+from rpython.jit.metainterp.optimizeopt.virtualstate import (
+    VirtualStateInfo, VStructStateInfo, LEVEL_CONSTANT, VArrayStateInfo,
+    not_virtual, VirtualState, GenerateGuardState, VirtualStatesCantMatch,
+    VArrayStructStateInfo, VirtualStateConstructor)
 from rpython.jit.metainterp.history import ConstInt, ConstPtr, TargetToken
-from rpython.jit.metainterp.resoperation import InputArgInt, InputArgRef,\
-     InputArgFloat
+from rpython.jit.metainterp.resoperation import (
+    InputArgInt, InputArgRef, InputArgFloat)
 from rpython.jit.backend.llgraph.runner import ArrayDescr
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.rtyper import rclass
-from rpython.jit.metainterp.optimizeopt.test.test_util import LLtypeMixin, BaseTest, \
-                                                           equaloplists
-from rpython.jit.metainterp.optimizeopt.intutils import IntBound,\
-     ConstIntBound, IntLowerBound, IntUpperBound, IntUnbounded
-from rpython.jit.metainterp.history import TreeLoop, JitCellToken
-from rpython.jit.metainterp.optimizeopt.test.test_util import FakeMetaInterpStaticData
+from rpython.jit.metainterp.optimizeopt.test.test_util import (
+    LLtypeMixin, BaseTest, equaloplists)
+from rpython.jit.metainterp.optimizeopt.intutils import (
+    IntBound, ConstIntBound, IntLowerBound, IntUnbounded)
+from rpython.jit.metainterp.history import JitCellToken
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer
 from rpython.jit.metainterp.resoperation import ResOperation, rop
-from rpython.jit.metainterp import resume, compile
+from rpython.jit.metainterp import compile
 from rpython.jit.metainterp.optimizeopt import info
 from rpython.jit.tool import oparser
 
@@ -99,14 +97,14 @@
         optimizer = FakeOptimizer(self.cpu)
         classbox1 = self.cpu.cls_of_box(InputArgRef(self.nodeaddr))
         innervalue1 = info.InstancePtrInfo(
-                known_class=classbox1, is_virtual=True,
-                descr=self.valuedescr.get_parent_descr())
+            known_class=classbox1, is_virtual=True,
+            descr=self.valuedescr.get_parent_descr())
         for field in self.valuedescr.get_parent_descr().get_all_fielddescrs():
             innervalue1.setfield(field, None, ConstInt(42))
         classbox2 = self.cpu.cls_of_box(InputArgRef(self.myptr3))
         innervalue2 = info.InstancePtrInfo(
-                known_class=classbox2, is_virtual=True,
-                descr=self.valuedescr3.get_parent_descr())
+            known_class=classbox2, is_virtual=True,
+            descr=self.valuedescr3.get_parent_descr())
         for field in self.valuedescr3.get_parent_descr().get_all_fielddescrs():
             innervalue2.setfield(field, None, ConstInt(42))
 
@@ -860,17 +858,14 @@
         if call_pure_results is not None:
             for k, v in call_pure_results.items():
                 bridge.call_pure_results[list(k)] = v
-        metainterp_sd = FakeMetaInterpStaticData(self.cpu)
-        metainterp_sd.virtualref_info = self.vrefinfo
-        #
-        trace = oparser.convert_loop_to_trace(bridge, metainterp_sd)
+        trace = oparser.convert_loop_to_trace(bridge, self.metainterp_sd)
 
         runtime_boxes = self.convert_values(bridge.operations[-1].getarglist(),
                                             values)
         data = compile.BridgeCompileData(trace, runtime_boxes,
             enable_opts=self.enable_opts, inline_short_preamble=True)
 
-        info, newops = data.optimize_trace(metainterp_sd, None, {})
+        info, newops = data.optimize_trace(self.metainterp_sd, None, {})
         if info.final():
             bridge.operations = newops
             bridge.inputargs = info.inputargs

From pypy.commits at gmail.com  Thu May 16 01:57:16 2019
From: pypy.commits at gmail.com (stevie_92)
Date: Wed, 15 May 2019 22:57:16 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-gc-cycle: Fixed tp_clear call in cpyext
 (GIL handling)
Message-ID: <5cdcfbbc.1c69fb81.b75f8.e16b@mx.google.com>

Author: Stefan Beyer <home at sbeyer.at>
Branch: cpyext-gc-cycle
Changeset: r96623:074fd35b8463
Date: 2019-05-16 07:56 +0200
http://bitbucket.org/pypy/pypy/changeset/074fd35b8463/

Log:	Fixed tp_clear call in cpyext (GIL handling) Fixed some inheritance
	issues with cpyext objects Added debug output for rrc lists WIP:
	implement gc support for cpyext tuples WIP: handling of untracked
	rrc objects in incminimark (keep flag, etc)

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1185,6 +1185,10 @@
     state.C._PyPy_tuple_free = rffi.llexternal(
         '_PyPy_tuple_free', [rffi.VOIDP], lltype.Void,
         compilation_info=eci, _nowrapper=True)
+    from pypy.module.cpyext.typeobjectdefs import visitproc
+    state.C._PyPy_tuple_traverse = rffi.llexternal(
+        '_PyPy_tuple_traverse', [PyObject, visitproc, rffi.VOIDP],
+        rffi.INT, compilation_info=eci, _nowrapper=True)
     state.C._PyPy_tuple_dealloc = rffi.llexternal(
         '_PyPy_tuple_dealloc', [PyObject], lltype.Void,
         compilation_info=eci, _nowrapper=True)
diff --git a/pypy/module/cpyext/include/tupleobject.h b/pypy/module/cpyext/include/tupleobject.h
--- a/pypy/module/cpyext/include/tupleobject.h
+++ b/pypy/module/cpyext/include/tupleobject.h
@@ -19,6 +19,7 @@
 PyAPI_FUNC(PyObject *) PyTuple_New(Py_ssize_t size);
 PyAPI_FUNC(void) _PyPy_tuple_dealloc(PyObject *);
 PyAPI_FUNC(void) _PyPy_tuple_free(void *);
+PyAPI_FUNC(int) _PyPy_tuple_traverse(PyObject *ob, visitproc visit, void *arg);
 
 /* defined in varargswrapper.c */
 PyAPI_FUNC(PyObject *) PyTuple_Pack(Py_ssize_t, ...);
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -81,6 +81,12 @@
         state = space.fromcache(State)
         return state.C.PyObject_Free
 
+    def has_traverse(self, space):
+        return False
+
+    def get_traverse(self, space):
+        return 0
+
     def allocate(self, space, w_type, itemcount=0, immortal=False):
         state = space.fromcache(State)
         ob_type = rffi.cast(PyTypeObjectPtr, as_pyobj(space, w_type))
@@ -129,6 +135,7 @@
     tp_realize    = kw.pop('realize', None)
     tp_dealloc    = kw.pop('dealloc', None)
     tp_free       = kw.pop('free', None)
+    tp_traverse   = kw.pop('traverse', None)
     assert not kw, "Extra arguments to make_typedescr"
 
     null_dealloc = lltype.nullptr(lltype.FuncType([PyObject], lltype.Void))
@@ -159,6 +166,17 @@
         if tp_realize:
             def realize(self, space, ref):
                 return tp_realize(space, ref)
+
+        if tp_traverse:
+            def get_traverse(self, space):
+                from rpython.rtyper.lltypesystem import llmemory
+                from pypy.module.cpyext.typeobjectdefs import traverseproc
+                ptr = rffi.cast(traverseproc, tp_traverse)
+                obj = llmemory.cast_ptr_to_adr(ptr)
+                return llmemory.cast_adr_to_int(obj, "symbolic")
+
+            def has_traverse(self, space):
+                return True
     if typedef:
         CpyTypedescr.__name__ = "CpyTypedescr_%s" % (typedef.name,)
 
diff --git a/pypy/module/cpyext/src/tupleobject.c b/pypy/module/cpyext/src/tupleobject.c
--- a/pypy/module/cpyext/src/tupleobject.c
+++ b/pypy/module/cpyext/src/tupleobject.c
@@ -94,4 +94,15 @@
 _PyPy_tuple_free(void *obj)
 {
     PyObject_GC_Del(obj);
+}
+
+int
+_PyPy_tuple_traverse(PyObject *ob, visitproc visit, void *arg)
+{
+    PyTupleObject *o = (PyTupleObject *)ob;
+    Py_ssize_t i;
+
+    for (i = Py_SIZE(o); --i >= 0; )
+        Py_VISIT(o->ob_item[i]);
+    return 0;
 }
\ No newline at end of file
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -6,6 +6,7 @@
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib.rdynload import DLLHANDLE
 from rpython.rlib import rawrefcount
+from rpython.rlib.debug import debug_print
 import sys
 
 
@@ -177,6 +178,7 @@
                 def _tp_traverse(pyobj_ptr, callback, args):
                     from pypy.module.cpyext.api import PyObject
                     from pypy.module.cpyext.typeobjectdefs import visitproc
+                    from pypy.module.cpyext.pyobject import cts
                     # convert to pointers with correct types (PyObject)
                     callback_addr = llmemory.cast_ptr_to_adr(callback)
                     callback_ptr = llmemory.cast_adr_to_ptr(callback_addr,
@@ -184,9 +186,31 @@
                     pyobj_addr = llmemory.cast_ptr_to_adr(pyobj_ptr)
                     pyobj = llmemory.cast_adr_to_ptr(pyobj_addr, PyObject)
                     # now call tp_traverse (if possible)
-                    if pyobj.c_ob_type and pyobj.c_ob_type.c_tp_traverse:
-                        pyobj.c_ob_type.c_tp_traverse(pyobj, callback_ptr,
-                                                      args)
+                    debug_print("rrc check traverse", pyobj)
+                    pto = pyobj.c_ob_type
+                    if pto and pto.c_tp_name:
+                        tp_name = pto.c_tp_name
+                        name = rffi.charp2str(cts.cast('char*', tp_name))
+                        debug_print("rrc try traverse", pyobj, ": type", pto,
+                                    ": name", name)
+                        pto2 = pto
+                        i = 1
+                        while pto2.c_tp_base:
+                            base = pto2.c_tp_base
+                            if base.c_tp_name:
+                                tp_name = base.c_tp_name
+                                name = rffi.charp2str(cts.cast('char*', tp_name))
+                                debug_print(" " * i * 3, "basetype",
+                                            base, ": name",
+                                            name, "traverse",
+                                            base.c_tp_traverse)
+                            else:
+                                debug_print(" " * i * 3, "unknown base")
+                            pto2 = base
+                            i += 1
+                    if pto and pto.c_tp_traverse:
+                        debug_print("rrc do traverse", pyobj)
+                        pto.c_tp_traverse(pyobj, callback_ptr, args)
 
                 self.tp_traverse = (lambda o, v, a:_tp_traverse(o, v, a))
 
@@ -308,7 +332,8 @@
 def _rawrefcount_perform(space):
     from pypy.interpreter.baseobjspace import W_Root
     from pypy.module.cpyext.pyobject import (PyObject, incref, decref,
-                                             finalize, from_ref)
+                                             finalize, from_ref, cts)
+    from pypy.module.cpyext.api import generic_cpy_call
 
     while True:
         py_obj = rawrefcount.next_dead(PyObject)
@@ -328,9 +353,15 @@
             break
         pyobj = rffi.cast(PyObject, py_obj)
         adr_int = llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(pyobj))
-        if pyobj.c_ob_type.c_tp_clear:
+        pto = pyobj.c_ob_type
+        if pto.c_tp_clear:
             incref(space, py_obj)
-            pyobj.c_ob_type.c_tp_clear(pyobj)
+            if pto and pto.c_tp_name:
+                tp_name = pto.c_tp_name
+                name = rffi.charp2str(cts.cast('char*', tp_name))
+                debug_print("tp_clear", pyobj, ": type", pto,
+                            ": name", name)
+            generic_cpy_call(space, pto.c_tp_clear, pyobj)
             decref(space, py_obj)
         head = rawrefcount.cyclic_garbage_head(PyObject)
         if adr_int == llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(head)):
diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py
--- a/pypy/module/cpyext/tupleobject.py
+++ b/pypy/module/cpyext/tupleobject.py
@@ -46,7 +46,8 @@
                    alloc=tuple_alloc,
                    dealloc=state.C._PyPy_tuple_dealloc,
                    realize=tuple_realize,
-                   free=state.C._PyPy_tuple_free)
+                   free=state.C._PyPy_tuple_free,
+                   traverse=state.C._PyPy_tuple_traverse)
 
 PyTuple_Check, PyTuple_CheckExact = build_type_checkers_flags("Tuple")
 
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -653,7 +653,13 @@
     else:
         pto.c_tp_free = pto.c_tp_base.c_tp_free
 
-    # TODO: traverse (for tuple)
+    from rpython.rtyper.lltypesystem import llmemory
+    from pypy.module.cpyext.typeobjectdefs import traverseproc
+    if typedescr.has_traverse(space):
+        traverse = typedescr.get_traverse(space)
+        obj = llmemory.cast_int_to_adr(traverse)
+        traverse_ptr = llmemory.cast_adr_to_ptr(obj, traverseproc)
+        pto.c_tp_traverse = traverse_ptr
 
     if builder.cpyext_type_init is not None:
         builder.cpyext_type_init.append((pto, w_type))
@@ -723,6 +729,10 @@
             pto.c_tp_init = base.c_tp_init
         if not pto.c_tp_alloc:
             pto.c_tp_alloc = base.c_tp_alloc
+        if not pto.c_tp_clear:
+            pto.c_tp_clear = base.c_tp_clear
+        if not pto.c_tp_traverse:
+            pto.c_tp_traverse = base.c_tp_traverse
         # XXX check for correct GC flags!
         if not pto.c_tp_free:
             pto.c_tp_free = base.c_tp_free
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -3119,6 +3119,7 @@
     RAWREFCOUNT_FINALIZER_LEGACY = 2
     RAWREFCOUNT_REFS_SHIFT = 1
     RAWREFCOUNT_REFS_MASK_FINALIZED = 1
+    RAWREFCOUNT_REFS_UNTRACKED = -2
 
     def _pyobj(self, pyobjaddr):
         return llmemory.cast_adr_to_ptr(pyobjaddr, self.PYOBJ_HDR_PTR)
@@ -3379,6 +3380,9 @@
             rc -= REFCNT_FROM_PYPY
             self._pyobj(pyobject).c_ob_pypy_link = 0
             if rc == 0:
+                #pygchdr = self.rrc_pyobj_as_gc(self._pyobj(pyobject))
+                #if pygchdr <> lltype.nullptr(self.PYOBJ_GC_HDR):
+                #    self._rrc_debug_check_list(pygchdr)
                 self.rrc_dealloc_pending.append(pyobject)
                 # an object with refcnt == 0 cannot stay around waiting
                 # for its deallocator to be called.  Some code (lxml)
@@ -3393,6 +3397,7 @@
     _rrc_free._always_inline_ = True
 
     def rrc_major_collection_trace(self):
+        self._rrc_debug_check_consistency(print_label="begin-mark")
         # Only trace and mark rawrefcounted object if we are not doing
         # something special, like building gc.garbage.
         if self.rrc_state == self.RAWREFCOUNT_STATE_DEFAULT:
@@ -3401,38 +3406,57 @@
                 self._rrc_check_finalizer()
             # collect all rawrefcounted roots
             self._rrc_collect_rawrefcount_roots(self.rrc_pyobj_list)
+            self._rrc_debug_check_consistency(print_label="roots-marked")
             # mark all objects reachable from rawrefcounted roots
             self._rrc_mark_rawrefcount()
+            self._rrc_debug_check_consistency(print_label="before-fin")
             self.rrc_state = self.RAWREFCOUNT_STATE_MARKING
             if self._rrc_find_garbage(): # handle legacy finalizers
                 self._rrc_mark_garbage()
+                self._rrc_debug_check_consistency(print_label="end-legacy-fin")
             self.rrc_state = self.RAWREFCOUNT_STATE_DEFAULT
             use_cylicrc = not self._rrc_find_finalizer() # modern finalizers
+            self._rrc_debug_check_consistency(print_label="end-mark-cyclic")
         else:
             use_cylicrc = False # don't sweep any objects in cyclic isolates
 
         # now mark all pypy objects at the border, depending on the results
+        debug_print("use_cylicrc", use_cylicrc)
         self.rrc_p_list_old.foreach(self._rrc_major_trace, use_cylicrc)
-        self.rrc_o_list_old.foreach(self._rrc_major_trace, use_cylicrc)
+
+        # TODO: check again, if some new border objects have been marked and
+        #       continue marking recursively... why needed? -> wrapper for
+        #       pypy-obj is no pygc-obj??? ...
+
+        self._rrc_debug_check_consistency(print_label="end-mark")
+        #self.rrc_o_list_old.foreach(self._rrc_major_trace, use_cylicrc) # TODO: remove?
 
     def _rrc_major_trace(self, pyobject, use_cylicrefcnt):
         from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY
         from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY_LIGHT
         #
+        pyobj = self._pyobj(pyobject)
+        cyclic_rc = -42
         if use_cylicrefcnt:
-            pygchdr = self.rrc_pyobj_as_gc(self._pyobj(pyobject))
+            pygchdr = self.rrc_pyobj_as_gc(pyobj)
             if pygchdr != lltype.nullptr(self.PYOBJ_GC_HDR):
+                #if pygchdr.c_gc_refs != self.RAWREFCOUNT_REFS_UNTRACKED:
                 rc = pygchdr.c_gc_refs >> self.RAWREFCOUNT_REFS_SHIFT
+                cyclic_rc = rc
+                #else:
+                #    rc = pyobj.c_ob_refcnt
             else:
-                rc = self._pyobj(pyobject).c_ob_refcnt
+                rc = pyobj.c_ob_refcnt
         else:
-            rc = self._pyobj(pyobject).c_ob_refcnt
+            rc = pyobj.c_ob_refcnt
 
         if rc == REFCNT_FROM_PYPY or rc == REFCNT_FROM_PYPY_LIGHT or rc == 0:
             pass  # the corresponding object may die
         else:
             # force the corresponding object to be alive
-            intobj = self._pyobj(pyobject).c_ob_pypy_link
+            debug_print("pyobj stays alive", pyobj, "rc", rc, "cyclic_rc",
+                        cyclic_rc)
+            intobj = pyobj.c_ob_pypy_link
             obj = llmemory.cast_int_to_adr(intobj)
             self.objects_to_trace.append(obj)
             self.visit_all_objects()
@@ -3445,7 +3469,7 @@
         new_p_list = self.AddressStack()
         while self.rrc_p_list_old.non_empty():
             self._rrc_major_free(self.rrc_p_list_old.pop(), new_p_list,
-                                                            new_p_dict)
+                                                              new_p_dict)
         self.rrc_p_list_old.delete()
         self.rrc_p_list_old = new_p_list
         #
@@ -3453,15 +3477,17 @@
         no_o_dict = self.null_address_dict()
         while self.rrc_o_list_old.non_empty():
             self._rrc_major_free(self.rrc_o_list_old.pop(), new_o_list,
-                                                            no_o_dict)
+                                                              no_o_dict)
         self.rrc_o_list_old.delete()
         self.rrc_o_list_old = new_o_list
 
         if self.rrc_state == self.RAWREFCOUNT_STATE_DEFAULT:
             if not self._rrc_gc_list_is_empty(self.rrc_pyobj_old_list):
+                self._rrc_debug_check_consistency()
                 self._rrc_clear_weakref_callbacks()
                 self._rrc_gc_list_merge(self.rrc_pyobj_old_list,
                                         self.rrc_pyobj_dead_list)
+                self._rrc_debug_check_consistency(print_label="before-sweep")
 
     def _rrc_clear_weakref_callbacks(self):
         # Look for any weakrefs within the trash cycle and remove the callback.
@@ -3527,13 +3553,13 @@
                 refcnt -= REFCNT_FROM_PYPY_LIGHT
             elif refcnt >= REFCNT_FROM_PYPY:
                 refcnt -= REFCNT_FROM_PYPY
-            pygchdr.c_gc_refs = pygchdr.c_gc_refs & \
-                                self.RAWREFCOUNT_REFS_MASK_FINALIZED
-            pygchdr.c_gc_refs = refcnt << self.RAWREFCOUNT_REFS_SHIFT
+            self._rrc_pyobj_gc_refcnt_set(pygchdr, refcnt)
             pygchdr = pygchdr.c_gc_next
 
+        self._rrc_debug_check_consistency(print_label="rc-initialized")
+
         # For every object in this set, if it is marked, add 1 as a real
-        # refcount
+        # refcount (p_list => pyobj stays alive if obj stays alive).
         self.rrc_p_list_old.foreach(self._rrc_obj_fix_refcnt, None)
         self.rrc_o_list_old.foreach(self._rrc_obj_fix_refcnt, None)
 
@@ -3548,13 +3574,31 @@
         # now all rawrefcounted roots or live border objects have a
         # refcount > 0
 
+    def _rrc_pyobj_gc_refcnt_set(self, pygchdr, refcnt):
+        pygchdr.c_gc_refs &= self.RAWREFCOUNT_REFS_MASK_FINALIZED
+        pygchdr.c_gc_refs |= refcnt << self.RAWREFCOUNT_REFS_SHIFT
+
     def _rrc_obj_fix_refcnt(self, pyobject, ignore):
-        intobj = self._pyobj(pyobject).c_ob_pypy_link
+        from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY
+        from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY_LIGHT
+        #
+        pyobj = self._pyobj(pyobject)
+        intobj = pyobj.c_ob_pypy_link
         obj = llmemory.cast_int_to_adr(intobj)
-        gchdr = self.rrc_pyobj_as_gc(self._pyobj(pyobject))
+        gchdr = self.rrc_pyobj_as_gc(pyobj)
         if gchdr <> lltype.nullptr(self.PYOBJ_GC_HDR):
-            if self.header(obj).tid & (GCFLAG_VISITED | GCFLAG_NO_HEAP_PTRS):
-                gchdr.c_gc_refs += 1 << self.RAWREFCOUNT_REFS_SHIFT
+            #if gchdr.c_gc_refs == self.RAWREFCOUNT_REFS_UNTRACKED:
+            #    debug_print("gc obj not tracked", gchdr, ": obj", obj)
+            #else:
+            refcnt = pyobj.c_ob_refcnt
+            if refcnt >= REFCNT_FROM_PYPY_LIGHT:
+                refcnt -= REFCNT_FROM_PYPY_LIGHT
+            elif refcnt >= REFCNT_FROM_PYPY:
+                refcnt -= REFCNT_FROM_PYPY
+            if self.header(obj).tid & (GCFLAG_VISITED |
+                                       GCFLAG_NO_HEAP_PTRS):
+                refcnt += 1
+            self._rrc_pyobj_gc_refcnt_set(gchdr, refcnt)
 
     def _rrc_mark_rawrefcount(self):
         if self._rrc_gc_list_is_empty(self.rrc_pyobj_list):
@@ -3785,3 +3829,50 @@
 
     def _rrc_gc_list_is_empty(self, pygclist):
         return pygclist.c_gc_next == pygclist
+
+    def _rrc_debug_check_consistency(self, print_label=None):
+        if self.DEBUG:
+            should_print = print_label is not None
+            if should_print:
+                debug_start("rrc-lists " + print_label)
+            self._rrc_debug_check_list(self.rrc_pyobj_list,
+                                       should_print, "rrc_pyobj_list")
+            self._rrc_debug_check_list(self.rrc_pyobj_old_list,
+                                       should_print, "rrc_pyobj_old_list")
+            self._rrc_debug_check_list(self.rrc_pyobj_dead_list,
+                                       should_print, "rrc_pyobj_dead_list")
+            self._rrc_debug_check_list(self.rrc_pyobj_garbage_list,
+                                       should_print, "rrc_pyobj_garbage_list")
+            self._rrc_debug_check_list(self.rrc_pyobj_isolate_list,
+                                       should_print, "rrc_pyobj_isolate_list")
+            if should_print:
+                debug_stop("rrc-lists " + print_label)
+
+    def _rrc_debug_check_list(self, list, should_print, print_label):
+        if should_print:
+            debug_start(print_label)
+        gchdr = list.c_gc_next
+        prev = list
+        while gchdr <> list:
+            if should_print:
+                pyobj = self.rrc_gc_as_pyobj(gchdr)
+                intobj = pyobj.c_ob_pypy_link
+                debug_print("item", gchdr, ": pyobj", pyobj,
+                            "cyclic refcnt",
+                            gchdr.c_gc_refs >> self.RAWREFCOUNT_REFS_SHIFT,
+                            "refcnt", pyobj.c_ob_refcnt,
+                            "link", intobj)
+                if intobj:
+                    obj = llmemory.cast_int_to_adr(intobj)
+                    marked = self.header(obj).tid & \
+                             (GCFLAG_VISITED | GCFLAG_NO_HEAP_PTRS)
+                    debug_print("  linked obj", obj, ": marked", marked)
+
+            ll_assert(gchdr.c_gc_next != lltype.nullptr(self.PYOBJ_GC_HDR),
+                      "gc_next is null")
+            ll_assert(gchdr.c_gc_prev == prev, "gc_prev is inconsistent")
+            prev = gchdr
+            gchdr = gchdr.c_gc_next
+        ll_assert(list.c_gc_prev == prev, "gc_prev is inconsistent")
+        if should_print:
+            debug_stop(print_label)
\ No newline at end of file

From pypy.commits at gmail.com  Thu May 16 02:28:48 2019
From: pypy.commits at gmail.com (arigo)
Date: Wed, 15 May 2019 23:28:48 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: Merged in djones6/pypy/dj_s390 (pull
 request #645)
Message-ID: <5cdd0320.1c69fb81.fe1a9.e166@mx.google.com>

Author: Armin Rigo <armin.rigo at gmail.com>
Branch: py3.6
Changeset: r96625:44324cb083f8
Date: 2019-05-16 06:28 +0000
http://bitbucket.org/pypy/pypy/changeset/44324cb083f8/

Log:	Merged in djones6/pypy/dj_s390 (pull request #645)

	Add s390x z14 support to gcc command

diff --git a/rpython/translator/platform/arch/s390x.py b/rpython/translator/platform/arch/s390x.py
old mode 100644
new mode 100755
--- a/rpython/translator/platform/arch/s390x.py
+++ b/rpython/translator/platform/arch/s390x.py
@@ -88,6 +88,8 @@
         return "zEC12"
     if machine == 0x2964:
         return "z13"
+    if machine == 0x3907:  # gcc supports z14 as of 2019/05/08
+	    return "z14"
 
     # well all others are unsupported!
     return "unknown"

From pypy.commits at gmail.com  Thu May 16 02:28:57 2019
From: pypy.commits at gmail.com (djones6)
Date: Wed, 15 May 2019 23:28:57 -0700 (PDT)
Subject: [pypy-commit] pypy dj_s390: Add s390x z14 support to gcc command
Message-ID: <5cdd0329.1c69fb81.ed8ab.6ae3@mx.google.com>

Author: DeVerne Jones <djones6 at unb.ca>
Branch: dj_s390
Changeset: r96624:df06edd62de5
Date: 2019-05-15 16:42 -0300
http://bitbucket.org/pypy/pypy/changeset/df06edd62de5/

Log:	Add s390x z14 support to gcc command

diff --git a/rpython/translator/platform/arch/s390x.py b/rpython/translator/platform/arch/s390x.py
old mode 100644
new mode 100755
--- a/rpython/translator/platform/arch/s390x.py
+++ b/rpython/translator/platform/arch/s390x.py
@@ -88,6 +88,8 @@
         return "zEC12"
     if machine == 0x2964:
         return "z13"
+    if machine == 0x3907:  # gcc supports z14 as of 2019/05/08
+	    return "z14"
 
     # well all others are unsupported!
     return "unknown"

From pypy.commits at gmail.com  Thu May 16 03:42:14 2019
From: pypy.commits at gmail.com (arigo)
Date: Thu, 16 May 2019 00:42:14 -0700 (PDT)
Subject: [pypy-commit] pypy default: back-port df06edd62de5: Add s390x z14
 support to gcc command
Message-ID: <5cdd1456.1c69fb81.a4197.56d9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r96626:d7d99b9b6d25
Date: 2019-05-16 09:41 +0200
http://bitbucket.org/pypy/pypy/changeset/d7d99b9b6d25/

Log:	back-port df06edd62de5: Add s390x z14 support to gcc command

diff --git a/rpython/translator/platform/arch/s390x.py b/rpython/translator/platform/arch/s390x.py
--- a/rpython/translator/platform/arch/s390x.py
+++ b/rpython/translator/platform/arch/s390x.py
@@ -88,6 +88,8 @@
         return "zEC12"
     if machine == 0x2964:
         return "z13"
+    if machine == 0x3907:  # gcc supports z14 as of 2019/05/08
+	    return "z14"
 
     # well all others are unsupported!
     return "unknown"

From pypy.commits at gmail.com  Thu May 16 23:35:12 2019
From: pypy.commits at gmail.com (mattip)
Date: Thu, 16 May 2019 20:35:12 -0700 (PDT)
Subject: [pypy-commit] pypy default: untabbify
Message-ID: <5cde2bf0.1c69fb81.3af40.776c@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r96627:db5598e252c6
Date: 2019-05-17 06:32 +0300
http://bitbucket.org/pypy/pypy/changeset/db5598e252c6/

Log:	untabbify

diff --git a/rpython/translator/platform/arch/s390x.py b/rpython/translator/platform/arch/s390x.py
--- a/rpython/translator/platform/arch/s390x.py
+++ b/rpython/translator/platform/arch/s390x.py
@@ -89,7 +89,7 @@
     if machine == 0x2964:
         return "z13"
     if machine == 0x3907:  # gcc supports z14 as of 2019/05/08
-	    return "z14"
+        return "z14"
 
     # well all others are unsupported!
     return "unknown"

From pypy.commits at gmail.com  Thu May 16 23:35:15 2019
From: pypy.commits at gmail.com (mattip)
Date: Thu, 16 May 2019 20:35:15 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: merge default into py.36
Message-ID: <5cde2bf3.1c69fb81.c4a54.6610@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r96628:c26316b6a81d
Date: 2019-05-17 06:34 +0300
http://bitbucket.org/pypy/pypy/changeset/c26316b6a81d/

Log:	merge default into py.36

diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst
--- a/pypy/doc/how-to-release.rst
+++ b/pypy/doc/how-to-release.rst
@@ -8,6 +8,12 @@
 a branch named like release-pypy3.5-v2.x or release-pypy3.5-v4.x, and each
 release is tagged, for instance release-pypy3.5-v4.0.1. 
 
+The release version number should be bumped. A micro release increment means
+there were no changes that justify rebuilding c-extension wheels, since
+the wheels are marked with only major.minor version numbers. It is ofen not
+clear what constitues a "major" release verses a "minor" release, the release
+manager can make that call.
+
 After release, inevitably there are bug fixes. It is the responsibility of
 the commiter who fixes a bug to make sure this fix is on the release branch,
 so that we can then create a tagged bug-fix release, which will hopefully
@@ -78,6 +84,8 @@
 
 * Maybe bump the SOABI number in module/imp/importing. This has many
   implications, so make sure the PyPy community agrees to the change.
+  Wheels will use the major.minor release numbers in the name, so bump
+  them if there is an incompatible change to cpyext.
 
 * Update and write documentation
 
diff --git a/rpython/translator/platform/arch/s390x.py b/rpython/translator/platform/arch/s390x.py
--- a/rpython/translator/platform/arch/s390x.py
+++ b/rpython/translator/platform/arch/s390x.py
@@ -89,7 +89,7 @@
     if machine == 0x2964:
         return "z13"
     if machine == 0x3907:  # gcc supports z14 as of 2019/05/08
-	    return "z14"
+        return "z14"
 
     # well all others are unsupported!
     return "unknown"

From pypy.commits at gmail.com  Fri May 17 03:18:46 2019
From: pypy.commits at gmail.com (stevie_92)
Date: Fri, 17 May 2019 00:18:46 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-gc-cycle: Added support for untracked
 objects in rawrefcount tests
Message-ID: <5cde6056.1c69fb81.1dee0.0b01@mx.google.com>

Author: Stefan Beyer <home at sbeyer.at>
Branch: cpyext-gc-cycle
Changeset: r96629:ec43b45f7cc4
Date: 2019-05-16 16:19 +0200
http://bitbucket.org/pypy/pypy/changeset/ec43b45f7cc4/

Log:	Added support for untracked objects in rawrefcount tests Fixed a bug
	in incminimark when using rawrefcounted modern finalizers

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -3401,11 +3401,19 @@
         # Only trace and mark rawrefcounted object if we are not doing
         # something special, like building gc.garbage.
         if self.rrc_state == self.RAWREFCOUNT_STATE_DEFAULT:
+            merged_old_list = False
             # check objects with finalizers from last collection cycle
             if not self._rrc_gc_list_is_empty(self.rrc_pyobj_old_list):
-                self._rrc_check_finalizer()
+                merged_old_list = self._rrc_check_finalizer()
             # collect all rawrefcounted roots
             self._rrc_collect_rawrefcount_roots(self.rrc_pyobj_list)
+            if merged_old_list:
+                # set all refcounts to zero for objects in dead list
+                # (might have been incremented) by fix_refcnt
+                gchdr = self.rrc_pyobj_dead_list.c_gc_next
+                while gchdr <> self.rrc_pyobj_dead_list:
+                    gchdr.c_gc_refs = 0
+                    gchdr = gchdr.c_gc_next
             self._rrc_debug_check_consistency(print_label="roots-marked")
             # mark all objects reachable from rawrefcounted roots
             self._rrc_mark_rawrefcount()
@@ -3740,10 +3748,12 @@
         if found_alive:
             self._rrc_gc_list_merge(self.rrc_pyobj_old_list,
                                     self.rrc_pyobj_list)
+            return False
         else:
             self._rrc_clear_weakref_callbacks()
             self._rrc_gc_list_merge(self.rrc_pyobj_old_list,
                                     self.rrc_pyobj_dead_list)
+            return True
 
     def _rrc_find_finalizer(self):
         found_finalizer = False
@@ -3841,10 +3851,11 @@
                                        should_print, "rrc_pyobj_old_list")
             self._rrc_debug_check_list(self.rrc_pyobj_dead_list,
                                        should_print, "rrc_pyobj_dead_list")
-            self._rrc_debug_check_list(self.rrc_pyobj_garbage_list,
-                                       should_print, "rrc_pyobj_garbage_list")
             self._rrc_debug_check_list(self.rrc_pyobj_isolate_list,
                                        should_print, "rrc_pyobj_isolate_list")
+            # rrc_pyobj_garbage_list is not a real list, it just marks the
+            # first and the last object in rrc_pyobj_list, which are garbage
+
             if should_print:
                 debug_stop("rrc-lists " + print_label)
 
diff --git a/rpython/memory/gc/test/dot/free_cpython_simple.dot b/rpython/memory/gc/test/dot/free_cpython_untracked_1.dot
copy from rpython/memory/gc/test/dot/free_cpython_simple.dot
copy to rpython/memory/gc/test/dot/free_cpython_untracked_1.dot
--- a/rpython/memory/gc/test/dot/free_cpython_simple.dot
+++ b/rpython/memory/gc/test/dot/free_cpython_untracked_1.dot
@@ -1,6 +1,8 @@
 digraph G {
     "a" [type=C, alive=n];
-    "b" [type=C, alive=n];
+    "b" [type=C, alive=n, tracked=n];
+    "c" [type=C, alive=n];
     "a" -> "b";
-    "b" -> "a";
+    "b" -> "c";
+    "c" -> "a"
 }
diff --git a/rpython/memory/gc/test/dot/keep_cpython_self.dot b/rpython/memory/gc/test/dot/keep_cpython_untracked_1.dot
copy from rpython/memory/gc/test/dot/keep_cpython_self.dot
copy to rpython/memory/gc/test/dot/keep_cpython_untracked_1.dot
--- a/rpython/memory/gc/test/dot/keep_cpython_self.dot
+++ b/rpython/memory/gc/test/dot/keep_cpython_untracked_1.dot
@@ -1,4 +1,8 @@
 digraph G {
     "a" [type=C, alive=y, ext_refcnt=1];
-    "a" -> "a";
+    "b" [type=C, alive=y, tracked=n];
+    "c" [type=C, alive=y];
+    "a" -> "b";
+    "b" -> "c";
+    "c" -> "a";
 }
diff --git a/rpython/memory/gc/test/test_rawrefcount.py b/rpython/memory/gc/test/test_rawrefcount.py
--- a/rpython/memory/gc/test/test_rawrefcount.py
+++ b/rpython/memory/gc/test/test_rawrefcount.py
@@ -12,6 +12,7 @@
 RAWREFCOUNT_FINALIZER_MODERN = IncMiniMark.RAWREFCOUNT_FINALIZER_MODERN
 RAWREFCOUNT_FINALIZER_LEGACY = IncMiniMark.RAWREFCOUNT_FINALIZER_LEGACY
 RAWREFCOUNT_FINALIZER_NONE = IncMiniMark.RAWREFCOUNT_FINALIZER_NONE
+RAWREFCOUNT_REFS_UNTRACKED = IncMiniMark.RAWREFCOUNT_REFS_UNTRACKED
 
 S = lltype.GcForwardReference()
 S.become(lltype.GcStruct('S',
@@ -136,7 +137,8 @@
 
         return p1, p1ref, check_alive
 
-    def _rawrefcount_pyobj(self, create_immortal=False, is_gc=True):
+    def _rawrefcount_pyobj(self, create_immortal=False, is_gc=True,
+                           tracked=True):
         r1 = lltype.malloc(PYOBJ_HDR, flavor='raw',
                            immortal=create_immortal)
         r1.c_ob_refcnt = 0
@@ -144,13 +146,7 @@
         r1addr = llmemory.cast_ptr_to_adr(r1)
 
         if is_gc:
-            r1gc = lltype.malloc(PYOBJ_GC_HDR, flavor='raw',
-                                 immortal=True)
-            r1gc.c_gc_next = self.pyobj_list
-            r1gc.c_gc_prev = self.pyobj_list.c_gc_prev
-            r1gc.c_gc_prev.c_gc_next = r1gc
-            self.pyobj_list.c_gc_prev = r1gc
-            self.gcobjs.append(r1gc)
+            self._rawrefcount_add_gc(tracked)
 
         self.pyobjs.append(r1)
         self.is_pygc.append(is_gc)
@@ -164,7 +160,8 @@
 
     def _rawrefcount_pair(self, intval, is_light=False, is_pyobj=False,
                           create_old=False, create_immortal=False,
-                          rooted=False, force_external=False, is_gc=True):
+                          rooted=False, force_external=False, is_gc=True,
+                          tracked=True):
         if is_light:
             rc = REFCNT_FROM_PYPY_LIGHT
         else:
@@ -197,13 +194,7 @@
         r1addr = llmemory.cast_ptr_to_adr(r1)
 
         if is_gc:
-            r1gc = lltype.malloc(PYOBJ_GC_HDR, flavor='raw',
-                                 immortal=True)
-            r1gc.c_gc_next = self.pyobj_list
-            r1gc.c_gc_prev = self.pyobj_list.c_gc_prev
-            r1gc.c_gc_prev.c_gc_next = r1gc
-            self.pyobj_list.c_gc_prev = r1gc
-            self.gcobjs.append(r1gc)
+            self._rawrefcount_add_gc(tracked)
 
         self.pyobjs.append(r1)
         self.is_pygc.append(is_gc)
@@ -231,6 +222,19 @@
             return p1
         return p1, p1ref, r1, r1addr, check_alive
 
+    def _rawrefcount_add_gc(self, tracked):
+        r1gc = lltype.malloc(PYOBJ_GC_HDR, flavor='raw',
+                             immortal=True)
+        self.gcobjs.append(r1gc)
+        if tracked:
+            r1gc.c_gc_refs = 0
+            r1gc.c_gc_next = self.pyobj_list
+            r1gc.c_gc_prev = self.pyobj_list.c_gc_prev
+            r1gc.c_gc_prev.c_gc_next = r1gc
+            self.pyobj_list.c_gc_prev = r1gc
+        else:
+            r1gc.c_gc_refs = RAWREFCOUNT_REFS_UNTRACKED
+
     def test_rawrefcount_objects_basic(self, old=False):
         p1, p1ref, r1, r1addr, check_alive = (
             self._rawrefcount_pair(42, is_light=True, create_old=old))
@@ -519,6 +523,7 @@
             rooted = attr['rooted'] == "y" if 'rooted' in attr else False
             ext_refcnt = int(attr['ext_refcnt']) if 'ext_refcnt' in attr else 0
             finalizer = attr['finalizer'] if 'finalizer' in attr else None
+            tracked = attr['tracked'] == "y" if 'tracked' in attr else True
             if finalizer == "modern":
                 finalizers = True
             resurrect = attr['resurrect'] if 'resurrect' in attr else None
@@ -527,7 +532,8 @@
             info = NodeInfo(type, alive, ext_refcnt, finalizer, resurrect,
                             delete, garbage)
             if type == "C":
-                r, raddr, check_alive = self._rawrefcount_pyobj()
+                r, raddr, check_alive = self._rawrefcount_pyobj(
+                    tracked=tracked)
                 r.c_ob_refcnt += ext_refcnt
                 nodes[name] = CPythonNode(r, raddr, check_alive, info)
             elif type == "P":
@@ -539,7 +545,7 @@
             elif type == "B":
                 p, pref, r, raddr, check_alive =\
                     self._rawrefcount_pair(42 + i, rooted=rooted,
-                                           create_old=True)
+                                           create_old=True, tracked=tracked)
                 r.c_ob_refcnt += ext_refcnt
                 nodes[name] = BorderNode(p, pref, r, raddr, check_alive, info)
                 i += 1

From pypy.commits at gmail.com  Fri May 17 03:18:48 2019
From: pypy.commits at gmail.com (stevie_92)
Date: Fri, 17 May 2019 00:18:48 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-gc-cycle: Fixed untracked objects in
 rawrefcount tests
Message-ID: <5cde6058.1c69fb81.f2e6d.ddc9@mx.google.com>

Author: Stefan Beyer <home at sbeyer.at>
Branch: cpyext-gc-cycle
Changeset: r96630:1c39776f6612
Date: 2019-05-16 17:09 +0200
http://bitbucket.org/pypy/pypy/changeset/1c39776f6612/

Log:	Fixed untracked objects in rawrefcount tests

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -3448,11 +3448,11 @@
         if use_cylicrefcnt:
             pygchdr = self.rrc_pyobj_as_gc(pyobj)
             if pygchdr != lltype.nullptr(self.PYOBJ_GC_HDR):
-                #if pygchdr.c_gc_refs != self.RAWREFCOUNT_REFS_UNTRACKED:
-                rc = pygchdr.c_gc_refs >> self.RAWREFCOUNT_REFS_SHIFT
-                cyclic_rc = rc
-                #else:
-                #    rc = pyobj.c_ob_refcnt
+                if pygchdr.c_gc_refs != self.RAWREFCOUNT_REFS_UNTRACKED:
+                    rc = pygchdr.c_gc_refs >> self.RAWREFCOUNT_REFS_SHIFT
+                    cyclic_rc = rc
+                else:
+                    rc = pyobj.c_ob_refcnt
             else:
                 rc = pyobj.c_ob_refcnt
         else:
@@ -3595,18 +3595,18 @@
         obj = llmemory.cast_int_to_adr(intobj)
         gchdr = self.rrc_pyobj_as_gc(pyobj)
         if gchdr <> lltype.nullptr(self.PYOBJ_GC_HDR):
-            #if gchdr.c_gc_refs == self.RAWREFCOUNT_REFS_UNTRACKED:
-            #    debug_print("gc obj not tracked", gchdr, ": obj", obj)
-            #else:
-            refcnt = pyobj.c_ob_refcnt
-            if refcnt >= REFCNT_FROM_PYPY_LIGHT:
-                refcnt -= REFCNT_FROM_PYPY_LIGHT
-            elif refcnt >= REFCNT_FROM_PYPY:
-                refcnt -= REFCNT_FROM_PYPY
-            if self.header(obj).tid & (GCFLAG_VISITED |
-                                       GCFLAG_NO_HEAP_PTRS):
-                refcnt += 1
-            self._rrc_pyobj_gc_refcnt_set(gchdr, refcnt)
+            if gchdr.c_gc_refs == self.RAWREFCOUNT_REFS_UNTRACKED:
+                debug_print("gc obj not tracked", gchdr, ": obj", obj)
+            else:
+                refcnt = pyobj.c_ob_refcnt
+                if refcnt >= REFCNT_FROM_PYPY_LIGHT:
+                    refcnt -= REFCNT_FROM_PYPY_LIGHT
+                elif refcnt >= REFCNT_FROM_PYPY:
+                    refcnt -= REFCNT_FROM_PYPY
+                if self.header(obj).tid & (GCFLAG_VISITED |
+                                           GCFLAG_NO_HEAP_PTRS):
+                    refcnt += 1
+                self._rrc_pyobj_gc_refcnt_set(gchdr, refcnt)
 
     def _rrc_mark_rawrefcount(self):
         if self._rrc_gc_list_is_empty(self.rrc_pyobj_list):
@@ -3779,8 +3779,9 @@
     def _rrc_visit_action(self, pyobj, ignore):
         pygchdr = self.rrc_pyobj_as_gc(pyobj)
         if pygchdr <> lltype.nullptr(self.PYOBJ_GC_HDR):
-            pygchdr.c_gc_refs += self.rrc_refcnt_add << \
-                                 self.RAWREFCOUNT_REFS_SHIFT
+            if pygchdr.c_gc_refs != self.RAWREFCOUNT_REFS_UNTRACKED:
+                pygchdr.c_gc_refs += self.rrc_refcnt_add << \
+                                     self.RAWREFCOUNT_REFS_SHIFT
 
     def _rrc_traverse(self, pyobj, refcnt_add):
         from rpython.rlib.objectmodel import we_are_translated
diff --git a/rpython/memory/gc/test/dot/free_cpython_untracked_1.dot b/rpython/memory/gc/test/dot/free_cpython_untracked_1.dot
--- a/rpython/memory/gc/test/dot/free_cpython_untracked_1.dot
+++ b/rpython/memory/gc/test/dot/free_cpython_untracked_1.dot
@@ -1,8 +1,8 @@
 digraph G {
     "a" [type=C, alive=n];
-    "b" [type=C, alive=n, tracked=n];
-    "c" [type=C, alive=n];
+    "b" [type=C, alive=n];
+    "c" [type=C, alive=n, tracked=n];
     "a" -> "b";
-    "b" -> "c";
-    "c" -> "a"
+    "b" -> "a";
+    "b" -> "c"
 }
diff --git a/rpython/memory/gc/test/dot/keep_cpython_untracked_1.dot b/rpython/memory/gc/test/dot/keep_cpython_untracked_1.dot
--- a/rpython/memory/gc/test/dot/keep_cpython_untracked_1.dot
+++ b/rpython/memory/gc/test/dot/keep_cpython_untracked_1.dot
@@ -1,8 +1,8 @@
 digraph G {
     "a" [type=C, alive=y, ext_refcnt=1];
-    "b" [type=C, alive=y, tracked=n];
-    "c" [type=C, alive=y];
+    "b" [type=C, alive=y];
+    "c" [type=C, alive=y, tracked=n];
     "a" -> "b";
+    "b" -> "a";
     "b" -> "c";
-    "c" -> "a";
 }
diff --git a/rpython/memory/gc/test/test_rawrefcount.py b/rpython/memory/gc/test/test_rawrefcount.py
--- a/rpython/memory/gc/test/test_rawrefcount.py
+++ b/rpython/memory/gc/test/test_rawrefcount.py
@@ -660,9 +660,10 @@
                     finalize_modern(pyobj)
                 if pyobj.c_ob_refcnt == 0:
                     gchdr = self.gc.rrc_pyobj_as_gc(pyobj)
-                    next = gchdr.c_gc_next
-                    next.c_gc_prev = gchdr.c_gc_prev
-                    gchdr.c_gc_prev.c_gc_next = next
+                    if gchdr.c_gc_refs != RAWREFCOUNT_REFS_UNTRACKED:
+                        next = gchdr.c_gc_next
+                        next.c_gc_prev = gchdr.c_gc_prev
+                        gchdr.c_gc_prev.c_gc_next = next
                     decref_children(pyobj)
                     self.pyobjs[self.pyobjs.index(pyobj)] = \
                         lltype.nullptr(PYOBJ_HDR_PTR.TO)

From pypy.commits at gmail.com  Fri May 17 03:18:50 2019
From: pypy.commits at gmail.com (stevie_92)
Date: Fri, 17 May 2019 00:18:50 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-gc-cycle: Fixed untracked rawrefcounted
 objects bug and improved debug output
Message-ID: <5cde605a.1c69fb81.6ebf7.2f6e@mx.google.com>

Author: Stefan Beyer <home at sbeyer.at>
Branch: cpyext-gc-cycle
Changeset: r96631:5621f2f5f876
Date: 2019-05-17 09:17 +0200
http://bitbucket.org/pypy/pypy/changeset/5621f2f5f876/

Log:	Fixed untracked rawrefcounted objects bug and improved debug output

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -3119,7 +3119,7 @@
     RAWREFCOUNT_FINALIZER_LEGACY = 2
     RAWREFCOUNT_REFS_SHIFT = 1
     RAWREFCOUNT_REFS_MASK_FINALIZED = 1
-    RAWREFCOUNT_REFS_UNTRACKED = -2
+    RAWREFCOUNT_REFS_UNTRACKED = -2 << RAWREFCOUNT_REFS_SHIFT
 
     def _pyobj(self, pyobjaddr):
         return llmemory.cast_adr_to_ptr(pyobjaddr, self.PYOBJ_HDR_PTR)
@@ -3595,10 +3595,15 @@
         obj = llmemory.cast_int_to_adr(intobj)
         gchdr = self.rrc_pyobj_as_gc(pyobj)
         if gchdr <> lltype.nullptr(self.PYOBJ_GC_HDR):
-            if gchdr.c_gc_refs == self.RAWREFCOUNT_REFS_UNTRACKED:
-                debug_print("gc obj not tracked", gchdr, ": obj", obj)
+            rc = gchdr.c_gc_refs
+            refcnt = pyobj.c_ob_refcnt
+            if rc == self.RAWREFCOUNT_REFS_UNTRACKED:
+                debug_print("gc obj not tracked", gchdr, ": obj", obj,
+                            "real-rc", refcnt, "cyclic-rc", rc)
             else:
-                refcnt = pyobj.c_ob_refcnt
+                debug_print("gc obj tracked", gchdr, ": obj", obj, "real-rc",
+                            refcnt, "cyclic-rc", rc, "gc-next",
+                            gchdr.c_gc_next, "gc-prev", gchdr.c_gc_prev)
                 if refcnt >= REFCNT_FROM_PYPY_LIGHT:
                     refcnt -= REFCNT_FROM_PYPY_LIGHT
                 elif refcnt >= REFCNT_FROM_PYPY:

From pypy.commits at gmail.com  Sun May 19 04:46:33 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 19 May 2019 01:46:33 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: add memoryview.obj which stores a
 reference, issue #3016
Message-ID: <5ce117e9.1c69fb81.25cd1.d55f@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r96632:40df81b2d706
Date: 2019-05-19 11:45 +0300
http://bitbucket.org/pypy/pypy/changeset/40df81b2d706/

Log:	add memoryview.obj which stores a reference, issue #3016

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -57,7 +57,9 @@
             w_object._check_released(space)
             return W_MemoryView.copy(w_object)
         view = space.buffer_w(w_object, space.BUF_FULL_RO)
-        return view.wrap(space)
+        mv = view.wrap(space)
+        mv.obj = w_object
+        return mv
 
     def _make_descr__cmp(name):
         def descr__cmp(self, space, w_other):
@@ -265,6 +267,10 @@
         # I've never seen anyone filling this field
         return space.newtuple([])
 
+    def w_get_obj(self, space):
+        self._check_released(space)
+        return self.obj
+
     def descr_repr(self, space):
         if self.view is None:
             return self.getrepr(space, 'released memory')
@@ -529,6 +535,7 @@
     shape       = GetSetProperty(W_MemoryView.w_get_shape),
     strides     = GetSetProperty(W_MemoryView.w_get_strides),
     suboffsets  = GetSetProperty(W_MemoryView.w_get_suboffsets),
+    obj         = GetSetProperty(W_MemoryView.w_get_obj),
     _pypy_raw_address = interp2app(W_MemoryView.descr_pypy_raw_address),
     )
 W_MemoryView.typedef.acceptable_as_base_class = False
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -62,13 +62,15 @@
         assert w.tobytes() == bytes(w) == b'geb'
 
     def test_memoryview_attrs(self):
-        v = memoryview(b"a"*100)
+        b = b"a"*100
+        v = memoryview(b)
         assert v.format == "B"
         assert v.itemsize == 1
         assert v.shape == (100,)
         assert v.ndim == 1
         assert v.strides == (1,)
         assert v.nbytes == 100
+        assert v.obj is b
 
     def test_suboffsets(self):
         v = memoryview(b"a"*100)

From pypy.commits at gmail.com  Sun May 19 11:56:19 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 19 May 2019 08:56:19 -0700 (PDT)
Subject: [pypy-commit] pypy vendor/stdlib-2.7.16: vendor in 2.7.16 with pip
 19, stub implementation of weakref._remove_dead_weakref
Message-ID: <5ce17ca3.1c69fb81.d387.c66b@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: vendor/stdlib-2.7.16
Changeset: r96633:1ef1b169d3cc
Date: 2019-05-19 18:54 +0300
http://bitbucket.org/pypy/pypy/changeset/1ef1b169d3cc/

Log:	vendor in 2.7.16 with pip 19, stub implementation of
	weakref._remove_dead_weakref

diff too long, truncating to 2000 out of 56242 lines

diff --git a/lib-python/2.7/HTMLParser.py b/lib-python/2.7/HTMLParser.py
--- a/lib-python/2.7/HTMLParser.py
+++ b/lib-python/2.7/HTMLParser.py
@@ -462,11 +462,12 @@
             else:
                 # Cannot use name2codepoint directly, because HTMLParser supports apos,
                 # which is not part of HTML 4
-                import htmlentitydefs
                 if HTMLParser.entitydefs is None:
-                    entitydefs = HTMLParser.entitydefs = {'apos':u"'"}
+                    import htmlentitydefs
+                    entitydefs = {'apos':u"'"}
                     for k, v in htmlentitydefs.name2codepoint.iteritems():
                         entitydefs[k] = unichr(v)
+                    HTMLParser.entitydefs = entitydefs
                 try:
                     return self.entitydefs[s]
                 except KeyError:
diff --git a/lib-python/2.7/SocketServer.py b/lib-python/2.7/SocketServer.py
--- a/lib-python/2.7/SocketServer.py
+++ b/lib-python/2.7/SocketServer.py
@@ -229,6 +229,9 @@
                 # shutdown request and wastes cpu at all other times.
                 r, w, e = _eintr_retry(select.select, [self], [], [],
                                        poll_interval)
+                # bpo-35017: shutdown() called during select(), exit immediately.
+                if self.__shutdown_request:
+                    break
                 if self in r:
                     self._handle_request_noblock()
         finally:
diff --git a/lib-python/2.7/UserDict.py b/lib-python/2.7/UserDict.py
--- a/lib-python/2.7/UserDict.py
+++ b/lib-python/2.7/UserDict.py
@@ -113,12 +113,8 @@
     def __iter__(self):
         return iter(self.data)
 
-try:
-    import _abcoll
-except ImportError:
-    pass    # e.g. no '_weakref' module on this pypy
-else:
-    _abcoll.MutableMapping.register(IterableUserDict)
+import _abcoll
+_abcoll.MutableMapping.register(IterableUserDict)
 
 
 class DictMixin:
diff --git a/lib-python/2.7/_pyio.py b/lib-python/2.7/_pyio.py
--- a/lib-python/2.7/_pyio.py
+++ b/lib-python/2.7/_pyio.py
@@ -1619,6 +1619,7 @@
         self.buffer.write(b)
         if self._line_buffering and (haslf or "\r" in s):
             self.flush()
+        self._set_decoded_chars('')
         self._snapshot = None
         if self._decoder:
             self._decoder.reset()
diff --git a/lib-python/2.7/_strptime.py b/lib-python/2.7/_strptime.py
--- a/lib-python/2.7/_strptime.py
+++ b/lib-python/2.7/_strptime.py
@@ -254,8 +254,8 @@
         # format directives (%m, etc.).
         regex_chars = re_compile(r"([\\.^$*+?\(\){}\[\]|])")
         format = regex_chars.sub(r"\\\1", format)
-        whitespace_replacement = re_compile('\s+')
-        format = whitespace_replacement.sub('\s+', format)
+        whitespace_replacement = re_compile(r'\s+')
+        format = whitespace_replacement.sub(r'\\s+', format)
         while '%' in format:
             directive_index = format.index('%')+1
             processed_format = "%s%s%s" % (processed_format,
diff --git a/lib-python/2.7/_threading_local.py b/lib-python/2.7/_threading_local.py
--- a/lib-python/2.7/_threading_local.py
+++ b/lib-python/2.7/_threading_local.py
@@ -57,11 +57,7 @@
 
   >>> class MyLocal(local):
   ...     number = 2
-  ...     initialized = False
   ...     def __init__(self, **kw):
-  ...         if self.initialized:
-  ...             raise SystemError('__init__ called too many times')
-  ...         self.initialized = True
   ...         self.__dict__.update(kw)
   ...     def squared(self):
   ...         return self.number ** 2
@@ -98,7 +94,7 @@
   >>> thread.start()
   >>> thread.join()
   >>> log
-  [[('color', 'red'), ('initialized', True)], 11]
+  [[('color', 'red')], 11]
 
 without affecting this thread's data:
 
@@ -155,7 +151,7 @@
         object.__setattr__(self, '_local__args', (args, kw))
         object.__setattr__(self, '_local__lock', RLock())
 
-        if (args or kw) and (cls.__init__ == object.__init__):
+        if (args or kw) and (cls.__init__ is object.__init__):
             raise TypeError("Initialization arguments are not supported")
 
         # We need to create the thread dict in anticipation of
diff --git a/lib-python/2.7/aifc.py b/lib-python/2.7/aifc.py
--- a/lib-python/2.7/aifc.py
+++ b/lib-python/2.7/aifc.py
@@ -288,6 +288,8 @@
     # _ssnd_chunk -- instantiation of a chunk class for the SSND chunk
     # _framesize -- size of one frame in the file
 
+    _file = None  # Set here since __del__ checks it
+
     def initfp(self, file):
         self._version = 0
         self._decomp = None
@@ -306,6 +308,7 @@
         else:
             raise Error, 'not an AIFF or AIFF-C file'
         self._comm_chunk_read = 0
+        self._ssnd_chunk = None
         while 1:
             self._ssnd_seek_needed = 1
             try:
@@ -341,10 +344,16 @@
             self._decomp.SetParams(params)
 
     def __init__(self, f):
-        if type(f) == type(''):
+        if isinstance(f, basestring):
             f = __builtin__.open(f, 'rb')
-        # else, assume it is an open file object already
-        self.initfp(f)
+            try:
+                self.initfp(f)
+            except:
+                f.close()
+                raise
+        else:
+            # assume it is an open file object already
+            self.initfp(f)
 
     #
     # User visible methods.
@@ -562,8 +571,10 @@
     # _datalength -- the size of the audio samples written to the header
     # _datawritten -- the size of the audio samples actually written
 
+    _file = None  # Set here since __del__ checks it
+
     def __init__(self, f):
-        if type(f) == type(''):
+        if isinstance(f, basestring):
             filename = f
             f = __builtin__.open(f, 'wb')
         else:
diff --git a/lib-python/2.7/argparse.py b/lib-python/2.7/argparse.py
--- a/lib-python/2.7/argparse.py
+++ b/lib-python/2.7/argparse.py
@@ -324,7 +324,11 @@
             if len(prefix) + len(usage) > text_width:
 
                 # break usage into wrappable parts
-                part_regexp = r'\(.*?\)+|\[.*?\]+|\S+'
+                part_regexp = (
+                    r'\(.*?\)+(?=\s|$)|'
+                    r'\[.*?\]+(?=\s|$)|'
+                    r'\S+'
+                )
                 opt_usage = format(optionals, groups)
                 pos_usage = format(positionals, groups)
                 opt_parts = _re.findall(part_regexp, opt_usage)
@@ -1793,19 +1797,7 @@
             # error if this argument is not allowed with other previously
             # seen arguments, assuming that actions that use the default
             # value don't really count as "present"
-
-            # XXX PyPy bug-to-bug compatibility: "is" on primitive types
-            # is not consistent in CPython.  We'll assume it is close
-            # enough for ints (which is true only for "small ints"), but
-            # for floats and longs and complexes we'll go for the option
-            # of forcing "is" to say False, like it usually does on
-            # CPython.  A fix is pending on CPython trunk
-            # (http://bugs.python.org/issue18943) but that might change
-            # the details of the semantics and so not be applied to 2.7.
-            # See the line AA below.
-
-            if (argument_values is not action.default or
-                    type(argument_values) in (float, long, complex)):  # AA
+            if argument_values is not action.default:
                 seen_non_default_actions.add(action)
                 for conflict_action in action_conflicts.get(action, []):
                     if conflict_action in seen_non_default_actions:
diff --git a/lib-python/2.7/asynchat.py b/lib-python/2.7/asynchat.py
--- a/lib-python/2.7/asynchat.py
+++ b/lib-python/2.7/asynchat.py
@@ -133,7 +133,7 @@
                 # no terminator, collect it all
                 self.collect_incoming_data (self.ac_in_buffer)
                 self.ac_in_buffer = ''
-            elif isinstance(terminator, int) or isinstance(terminator, long):
+            elif isinstance(terminator, (int, long)):
                 # numeric terminator
                 n = terminator
                 if lb < n:
diff --git a/lib-python/2.7/asyncore.py b/lib-python/2.7/asyncore.py
--- a/lib-python/2.7/asyncore.py
+++ b/lib-python/2.7/asyncore.py
@@ -633,7 +633,11 @@
         write = send
 
         def close(self):
-            os.close(self.fd)
+            if self.fd < 0:
+                return
+            fd = self.fd
+            self.fd = -1
+            os.close(fd)
 
         def fileno(self):
             return self.fd
diff --git a/lib-python/2.7/bsddb/test/test_associate.py b/lib-python/2.7/bsddb/test/test_associate.py
--- a/lib-python/2.7/bsddb/test/test_associate.py
+++ b/lib-python/2.7/bsddb/test/test_associate.py
@@ -114,6 +114,22 @@
             dupDB.close()
             self.fail("DBError exception was expected")
 
+    @unittest.skipUnless(db.version() >= (4, 6), 'Needs 4.6+')
+    def test_associateListError(self):
+        db1 = db.DB(self.env)
+        db1.open('bad.db', "a.db", db.DB_BTREE, db.DB_CREATE)
+        db2 = db.DB(self.env)
+        db2.open('bad.db', "b.db", db.DB_BTREE, db.DB_CREATE)
+
+        db1.associate(db2, lambda a, b: [0])
+
+        msg = "TypeError: The list returned by DB->associate callback" \
+              " should be a list of strings."
+        with test_support.captured_output("stderr") as s:
+            db1.put("0", "1")
+        db1.close()
+        db2.close()
+        self.assertEquals(s.getvalue().strip(), msg)
 
 
 #----------------------------------------------------------------------
@@ -233,7 +249,7 @@
         self.assertEqual(vals, None, vals)
 
         vals = secDB.pget('Unknown', txn=txn)
-        self.assertTrue(vals[0] == 99 or vals[0] == '99', vals)
+        self.assertIn(vals[0], (99, '99'), vals)
         vals[1].index('Unknown')
         vals[1].index('Unnamed')
         vals[1].index('unknown')
@@ -247,7 +263,8 @@
             if type(self.keytype) == type(''):
                 self.assertTrue(int(rec[0]))  # for primary db, key is a number
             else:
-                self.assertTrue(rec[0] and type(rec[0]) == type(0))
+                self.assertTrue(rec[0])
+                self.assertIs(type(rec[0]), int)
             count = count + 1
             if verbose:
                 print rec
@@ -262,7 +279,7 @@
 
         # test cursor pget
         vals = self.cur.pget('Unknown', flags=db.DB_LAST)
-        self.assertTrue(vals[1] == 99 or vals[1] == '99', vals)
+        self.assertIn(vals[1], (99, '99'), vals)
         self.assertEqual(vals[0], 'Unknown')
         vals[2].index('Unknown')
         vals[2].index('Unnamed')
diff --git a/lib-python/2.7/bsddb/test/test_basics.py b/lib-python/2.7/bsddb/test/test_basics.py
--- a/lib-python/2.7/bsddb/test/test_basics.py
+++ b/lib-python/2.7/bsddb/test/test_basics.py
@@ -597,7 +597,7 @@
 
         d.put("abcde", "ABCDE");
         num = d.truncate()
-        self.assertTrue(num >= 1, "truncate returned <= 0 on non-empty database")
+        self.assertGreaterEqual(num, 1, "truncate returned <= 0 on non-empty database")
         num = d.truncate()
         self.assertEqual(num, 0,
                 "truncate on empty DB returned nonzero (%r)" % (num,))
@@ -616,9 +616,9 @@
     if db.version() >= (4, 6):
         def test08_exists(self) :
             self.d.put("abcde", "ABCDE")
-            self.assertTrue(self.d.exists("abcde") == True,
+            self.assertEqual(self.d.exists("abcde"), True,
                     "DB->exists() returns wrong value")
-            self.assertTrue(self.d.exists("x") == False,
+            self.assertEqual(self.d.exists("x"), False,
                     "DB->exists() returns wrong value")
 
     #----------------------------------------
@@ -773,7 +773,7 @@
             if verbose:
                 print 'log file: ' + log
             logs = self.env.log_archive(db.DB_ARCH_REMOVE)
-            self.assertTrue(not logs)
+            self.assertFalse(logs)
 
         self.txn = self.env.txn_begin()
 
@@ -785,9 +785,9 @@
             self.d.put("abcde", "ABCDE", txn=txn)
             txn.commit()
             txn = self.env.txn_begin()
-            self.assertTrue(self.d.exists("abcde", txn=txn) == True,
+            self.assertEqual(self.d.exists("abcde", txn=txn), True,
                     "DB->exists() returns wrong value")
-            self.assertTrue(self.d.exists("x", txn=txn) == False,
+            self.assertEqual(self.d.exists("x", txn=txn), False,
                     "DB->exists() returns wrong value")
             txn.abort()
 
@@ -802,7 +802,7 @@
         d.put("abcde", "ABCDE");
         txn = self.env.txn_begin()
         num = d.truncate(txn)
-        self.assertTrue(num >= 1, "truncate returned <= 0 on non-empty database")
+        self.assertGreaterEqual(num, 1, "truncate returned <= 0 on non-empty database")
         num = d.truncate(txn)
         self.assertEqual(num, 0,
                 "truncate on empty DB returned nonzero (%r)" % (num,))
@@ -1086,7 +1086,7 @@
         a = "example of private object"
         self.obj.set_private(a)
         b = self.obj.get_private()
-        self.assertTrue(a is b)  # Object identity
+        self.assertIs(a, b)  # Object identity
 
     def test03_leak_assignment(self) :
         a = "example of private object"
diff --git a/lib-python/2.7/bsddb/test/test_dbenv.py b/lib-python/2.7/bsddb/test/test_dbenv.py
--- a/lib-python/2.7/bsddb/test/test_dbenv.py
+++ b/lib-python/2.7/bsddb/test/test_dbenv.py
@@ -54,15 +54,15 @@
                 self.env.set_cache_max(0, size)
                 size2 = self.env.get_cache_max()
                 self.assertEqual(0, size2[0])
-                self.assertTrue(size <= size2[1])
-                self.assertTrue(2*size > size2[1])
+                self.assertLessEqual(size, size2[1])
+                self.assertGreater(2*size, size2[1])
 
     if db.version() >= (4, 4) :
         def test_mutex_stat(self) :
             self.env.open(self.homeDir, db.DB_CREATE | db.DB_INIT_MPOOL |
                     db.DB_INIT_LOCK)
             stat = self.env.mutex_stat()
-            self.assertTrue("mutex_inuse_max" in stat)
+            self.assertIn("mutex_inuse_max", stat)
 
         def test_lg_filemode(self) :
             for i in [0600, 0660, 0666] :
@@ -128,8 +128,8 @@
                 i = i*1024*1024
                 self.env.set_lg_regionmax(i)
                 j = self.env.get_lg_regionmax()
-                self.assertTrue(i <= j)
-                self.assertTrue(2*i > j)
+                self.assertLessEqual(i, j)
+                self.assertGreater(2*i, j)
 
         def test_lk_detect(self) :
             flags= [db.DB_LOCK_DEFAULT, db.DB_LOCK_EXPIRE, db.DB_LOCK_MAXLOCKS,
@@ -150,10 +150,10 @@
         def test_lg_bsize(self) :
             log_size = 70*1024
             self.env.set_lg_bsize(log_size)
-            self.assertTrue(self.env.get_lg_bsize() >= log_size)
-            self.assertTrue(self.env.get_lg_bsize() < 4*log_size)
+            self.assertGreaterEqual(self.env.get_lg_bsize(), log_size)
+            self.assertLess(self.env.get_lg_bsize(), 4*log_size)
             self.env.set_lg_bsize(4*log_size)
-            self.assertTrue(self.env.get_lg_bsize() >= 4*log_size)
+            self.assertGreaterEqual(self.env.get_lg_bsize(), 4*log_size)
 
         def test_setget_data_dirs(self) :
             dirs = ("a", "b", "c", "d")
@@ -185,7 +185,7 @@
             self.assertEqual(cachesize2[0], cachesize3[0])
             self.assertEqual(cachesize2[2], cachesize3[2])
             # In Berkeley DB 5.1, the cachesize can change when opening the Env
-            self.assertTrue(cachesize2[1] <= cachesize3[1])
+            self.assertLessEqual(cachesize2[1], cachesize3[1])
 
         def test_set_cachesize_dbenv_db(self) :
             # You can not configure the cachesize using
@@ -299,7 +299,7 @@
             msg = "This is a test..."
             self.env.log_printf(msg)
             logc = self.env.log_cursor()
-            self.assertTrue(msg in (logc.last()[1]))
+            self.assertIn(msg, logc.last()[1])
 
     if db.version() >= (4, 7) :
         def test_log_config(self) :
@@ -341,21 +341,21 @@
             txn.commit()
             logc = self.env.log_cursor()
             logc.last()  # Skip the commit
-            self.assertTrue(msg in (logc.prev()[1]))
+            self.assertIn(msg, logc.prev()[1])
 
             msg = "This is another test..."
             txn = self.env.txn_begin()
             self.env.log_printf(msg, txn=txn)
             txn.abort()  # Do not store the new message
             logc.last()  # Skip the abort
-            self.assertTrue(msg not in (logc.prev()[1]))
+            self.assertNotIn(msg, logc.prev()[1])
 
             msg = "This is a third test..."
             txn = self.env.txn_begin()
             self.env.log_printf(msg, txn=txn)
             txn.commit()  # Do not store the new message
             logc.last()  # Skip the commit
-            self.assertTrue(msg in (logc.prev()[1]))
+            self.assertIn(msg, logc.prev()[1])
 
 
 class DBEnv_memp(DBEnv):
@@ -372,39 +372,39 @@
 
     def test_memp_1_trickle(self) :
         self.db.put("hi", "bye")
-        self.assertTrue(self.env.memp_trickle(100) > 0)
+        self.assertGreater(self.env.memp_trickle(100), 0)
 
 # Preserve the order, do "memp_trickle" test first
     def test_memp_2_sync(self) :
         self.db.put("hi", "bye")
         self.env.memp_sync()  # Full flush
         # Nothing to do...
-        self.assertTrue(self.env.memp_trickle(100) == 0)
+        self.assertEqual(self.env.memp_trickle(100), 0)
 
         self.db.put("hi", "bye2")
         self.env.memp_sync((1, 0))  # NOP, probably
         # Something to do... or not
-        self.assertTrue(self.env.memp_trickle(100) >= 0)
+        self.assertGreaterEqual(self.env.memp_trickle(100), 0)
 
         self.db.put("hi", "bye3")
         self.env.memp_sync((123, 99))  # Full flush
         # Nothing to do...
-        self.assertTrue(self.env.memp_trickle(100) == 0)
+        self.assertEqual(self.env.memp_trickle(100), 0)
 
     def test_memp_stat_1(self) :
         stats = self.env.memp_stat()  # No param
-        self.assertTrue(len(stats)==2)
-        self.assertTrue("cache_miss" in stats[0])
+        self.assertEqual(len(stats), 2)
+        self.assertIn("cache_miss", stats[0])
         stats = self.env.memp_stat(db.DB_STAT_CLEAR)  # Positional param
-        self.assertTrue("cache_miss" in stats[0])
+        self.assertIn("cache_miss", stats[0])
         stats = self.env.memp_stat(flags=0)  # Keyword param
-        self.assertTrue("cache_miss" in stats[0])
+        self.assertIn("cache_miss", stats[0])
 
     def test_memp_stat_2(self) :
         stats=self.env.memp_stat()[1]
-        self.assertTrue(len(stats))==1
-        self.assertTrue("test" in stats)
-        self.assertTrue("page_in" in stats["test"])
+        self.assertEqual(len(stats), 1)
+        self.assertIn("test", stats)
+        self.assertIn("page_in", stats["test"])
 
 class DBEnv_logcursor(DBEnv):
     def setUp(self):
@@ -426,28 +426,28 @@
         DBEnv.tearDown(self)
 
     def _check_return(self, value) :
-        self.assertTrue(isinstance(value, tuple))
+        self.assertIsInstance(value, tuple)
         self.assertEqual(len(value), 2)
-        self.assertTrue(isinstance(value[0], tuple))
+        self.assertIsInstance(value[0], tuple)
         self.assertEqual(len(value[0]), 2)
-        self.assertTrue(isinstance(value[0][0], int))
-        self.assertTrue(isinstance(value[0][1], int))
-        self.assertTrue(isinstance(value[1], str))
+        self.assertIsInstance(value[0][0], int)
+        self.assertIsInstance(value[0][1], int)
+        self.assertIsInstance(value[1], str)
 
     # Preserve test order
     def test_1_first(self) :
         logc = self.env.log_cursor()
         v = logc.first()
         self._check_return(v)
-        self.assertTrue((1, 1) < v[0])
-        self.assertTrue(len(v[1])>0)
+        self.assertLess((1, 1), v[0])
+        self.assertGreater(len(v[1]), 0)
 
     def test_2_last(self) :
         logc = self.env.log_cursor()
         lsn_first = logc.first()[0]
         v = logc.last()
         self._check_return(v)
-        self.assertTrue(lsn_first < v[0])
+        self.assertLess(lsn_first, v[0])
 
     def test_3_next(self) :
         logc = self.env.log_cursor()
@@ -456,16 +456,16 @@
         lsn_first = logc.first()[0]
         v = logc.next()
         self._check_return(v)
-        self.assertTrue(lsn_first < v[0])
-        self.assertTrue(lsn_last > v[0])
+        self.assertLess(lsn_first, v[0])
+        self.assertGreater(lsn_last, v[0])
 
         v2 = logc.next()
-        self.assertTrue(v2[0] > v[0])
-        self.assertTrue(lsn_last > v2[0])
+        self.assertGreater(v2[0], v[0])
+        self.assertGreater(lsn_last, v2[0])
 
         v3 = logc.next()
-        self.assertTrue(v3[0] > v2[0])
-        self.assertTrue(lsn_last > v3[0])
+        self.assertGreater(v3[0], v2[0])
+        self.assertGreater(lsn_last, v3[0])
 
     def test_4_prev(self) :
         logc = self.env.log_cursor()
@@ -474,16 +474,16 @@
         lsn_last = logc.last()[0]
         v = logc.prev()
         self._check_return(v)
-        self.assertTrue(lsn_first < v[0])
-        self.assertTrue(lsn_last > v[0])
+        self.assertLess(lsn_first, v[0])
+        self.assertGreater(lsn_last, v[0])
 
         v2 = logc.prev()
-        self.assertTrue(v2[0] < v[0])
-        self.assertTrue(lsn_first < v2[0])
+        self.assertLess(v2[0], v[0])
+        self.assertLess(lsn_first, v2[0])
 
         v3 = logc.prev()
-        self.assertTrue(v3[0] < v2[0])
-        self.assertTrue(lsn_first < v3[0])
+        self.assertLess(v3[0], v2[0])
+        self.assertLess(lsn_first, v3[0])
 
     def test_5_current(self) :
         logc = self.env.log_cursor()
diff --git a/lib-python/2.7/bsddb/test/test_dbshelve.py b/lib-python/2.7/bsddb/test/test_dbshelve.py
--- a/lib-python/2.7/bsddb/test/test_dbshelve.py
+++ b/lib-python/2.7/bsddb/test/test_dbshelve.py
@@ -248,7 +248,7 @@
             self.assertEqual(value.L, [x] * 10)
 
         else:
-            self.assertTrue(0, 'Unknown key type, fix the test')
+            self.fail('Unknown key type, fix the test')
 
 #----------------------------------------------------------------------
 
diff --git a/lib-python/2.7/bsddb/test/test_dbtables.py b/lib-python/2.7/bsddb/test/test_dbtables.py
--- a/lib-python/2.7/bsddb/test/test_dbtables.py
+++ b/lib-python/2.7/bsddb/test/test_dbtables.py
@@ -82,8 +82,8 @@
             colval = pickle.loads(values[0][colname])
         else :
             colval = pickle.loads(bytes(values[0][colname], "iso8859-1"))
-        self.assertTrue(colval > 3.141)
-        self.assertTrue(colval < 3.142)
+        self.assertGreater(colval, 3.141)
+        self.assertLess(colval, 3.142)
 
 
     def test02(self):
diff --git a/lib-python/2.7/bsddb/test/test_distributed_transactions.py b/lib-python/2.7/bsddb/test/test_distributed_transactions.py
--- a/lib-python/2.7/bsddb/test/test_distributed_transactions.py
+++ b/lib-python/2.7/bsddb/test/test_distributed_transactions.py
@@ -79,7 +79,7 @@
         recovered_txns=self.dbenv.txn_recover()
         self.assertEqual(self.num_txns,len(recovered_txns))
         for gid,txn in recovered_txns :
-            self.assertTrue(gid in txns)
+            self.assertIn(gid, txns)
         del txn
         del recovered_txns
 
@@ -122,7 +122,7 @@
     # Be sure there are not pending transactions.
     # Check also database size.
         recovered_txns=self.dbenv.txn_recover()
-        self.assertTrue(len(recovered_txns)==0)
+        self.assertEqual(len(recovered_txns), 0)
         self.assertEqual(len(committed_txns),self.db.stat()["nkeys"])
 
 class DBTxn_distributedSYNC(DBTxn_distributed):
diff --git a/lib-python/2.7/bsddb/test/test_lock.py b/lib-python/2.7/bsddb/test/test_lock.py
--- a/lib-python/2.7/bsddb/test/test_lock.py
+++ b/lib-python/2.7/bsddb/test/test_lock.py
@@ -2,6 +2,7 @@
 TestCases for testing the locking sub-system.
 """
 
+import sys
 import time
 
 import unittest
@@ -10,7 +11,6 @@
 
 if have_threads :
     from threading import Thread
-    import sys
     if sys.version_info[0] < 3 :
         from threading import currentThread
     else :
@@ -129,7 +129,14 @@
         end_time=time.time()
         deadlock_detection.end=True
         # Floating point rounding
-        self.assertTrue((end_time-start_time) >= 0.0999)
+        if sys.platform == 'win32':
+            # bpo-30850: On Windows, tolerate 50 ms whereas 100 ms is expected.
+            # The lock sometimes times out after only 58 ms. Windows clocks
+            # have a bad resolution and bad accuracy.
+            min_dt = 0.050
+        else:
+            min_dt = 0.0999
+        self.assertGreaterEqual(end_time-start_time, min_dt)
         self.env.lock_put(lock)
         t.join()
 
@@ -137,7 +144,7 @@
         self.env.lock_id_free(anID2)
 
         if db.version() >= (4,6):
-            self.assertTrue(deadlock_detection.count>0)
+            self.assertGreater(deadlock_detection.count, 0)
 
     def theThread(self, lockType):
         import sys
diff --git a/lib-python/2.7/bsddb/test/test_misc.py b/lib-python/2.7/bsddb/test/test_misc.py
--- a/lib-python/2.7/bsddb/test/test_misc.py
+++ b/lib-python/2.7/bsddb/test/test_misc.py
@@ -25,7 +25,7 @@
     def test02_db_home(self):
         env = db.DBEnv()
         # check for crash fixed when db_home is used before open()
-        self.assertTrue(env.db_home is None)
+        self.assertIsNone(env.db_home)
         env.open(self.homeDir, db.DB_CREATE)
         if sys.version_info[0] < 3 :
             self.assertEqual(self.homeDir, env.db_home)
diff --git a/lib-python/2.7/bsddb/test/test_recno.py b/lib-python/2.7/bsddb/test/test_recno.py
--- a/lib-python/2.7/bsddb/test/test_recno.py
+++ b/lib-python/2.7/bsddb/test/test_recno.py
@@ -18,7 +18,7 @@
         def assertIsInstance(self, obj, datatype, msg=None) :
             return self.assertEqual(type(obj), datatype, msg=msg)
         def assertGreaterEqual(self, a, b, msg=None) :
-            return self.assertTrue(a>=b, msg=msg)
+            return self.assertGreaterEqual(a, b, msg=msg)
 
 
     def setUp(self):
diff --git a/lib-python/2.7/bsddb/test/test_replication.py b/lib-python/2.7/bsddb/test/test_replication.py
--- a/lib-python/2.7/bsddb/test/test_replication.py
+++ b/lib-python/2.7/bsddb/test/test_replication.py
@@ -186,20 +186,18 @@
         d = d.values()[0]  # There is only one
         self.assertEqual(d[0], "127.0.0.1")
         self.assertEqual(d[1], client_port)
-        self.assertTrue((d[2]==db.DB_REPMGR_CONNECTED) or \
-                (d[2]==db.DB_REPMGR_DISCONNECTED))
+        self.assertIn(d[2], (db.DB_REPMGR_CONNECTED, db.DB_REPMGR_DISCONNECTED))
 
         d = self.dbenvClient.repmgr_site_list()
         self.assertEqual(len(d), 1)
         d = d.values()[0]  # There is only one
         self.assertEqual(d[0], "127.0.0.1")
         self.assertEqual(d[1], master_port)
-        self.assertTrue((d[2]==db.DB_REPMGR_CONNECTED) or \
-                (d[2]==db.DB_REPMGR_DISCONNECTED))
+        self.assertIn(d[2], (db.DB_REPMGR_CONNECTED, db.DB_REPMGR_DISCONNECTED))
 
         if db.version() >= (4,6) :
             d = self.dbenvMaster.repmgr_stat(flags=db.DB_STAT_CLEAR);
-            self.assertTrue("msgs_queued" in d)
+            self.assertIn("msgs_queued", d)
 
         self.dbMaster=db.DB(self.dbenvMaster)
         txn=self.dbenvMaster.txn_begin()
@@ -247,7 +245,7 @@
         if time.time()>=timeout and startup_timeout:
             self.skipTest("replication test skipped due to random failure, "
                 "see issue 3892")
-        self.assertTrue(time.time()<timeout)
+        self.assertLess(time.time(), timeout)
         self.assertEqual("123", v)
 
         txn=self.dbenvMaster.txn_begin()
@@ -260,7 +258,7 @@
             txn.commit()
             if v is None :
                 time.sleep(0.02)
-        self.assertTrue(time.time()<timeout)
+        self.assertLess(time.time(), timeout)
         self.assertEqual(None, v)
 
 class DBBaseReplication(DBReplication) :
@@ -381,7 +379,7 @@
         while (time.time()<timeout) and not (self.confirmed_master and
                 self.client_startupdone) :
             time.sleep(0.02)
-        self.assertTrue(time.time()<timeout)
+        self.assertLess(time.time(), timeout)
 
         self.dbMaster=db.DB(self.dbenvMaster)
         txn=self.dbenvMaster.txn_begin()
@@ -410,7 +408,7 @@
             break
 
         d = self.dbenvMaster.rep_stat(flags=db.DB_STAT_CLEAR);
-        self.assertTrue("master_changes" in d)
+        self.assertIn("master_changes", d)
 
         txn=self.dbenvMaster.txn_begin()
         self.dbMaster.put("ABC", "123", txn=txn)
@@ -424,7 +422,7 @@
             txn.commit()
             if v is None :
                 time.sleep(0.02)
-        self.assertTrue(time.time()<timeout)
+        self.assertLess(time.time(), timeout)
         self.assertEqual("123", v)
 
         txn=self.dbenvMaster.txn_begin()
@@ -437,7 +435,7 @@
             txn.commit()
             if v is None :
                 time.sleep(0.02)
-        self.assertTrue(time.time()<timeout)
+        self.assertLess(time.time(), timeout)
         self.assertEqual(None, v)
 
     if db.version() >= (4,7) :
diff --git a/lib-python/2.7/bsddb/test/test_sequence.py b/lib-python/2.7/bsddb/test/test_sequence.py
--- a/lib-python/2.7/bsddb/test/test_sequence.py
+++ b/lib-python/2.7/bsddb/test/test_sequence.py
@@ -82,7 +82,7 @@
         stat = self.seq.stat()
         for param in ('nowait', 'min', 'max', 'value', 'current',
                       'flags', 'cache_size', 'last_value', 'wait'):
-            self.assertTrue(param in stat, "parameter %s isn't in stat info" % param)
+            self.assertIn(param, stat, "parameter %s isn't in stat info" % param)
 
     if db.version() >= (4,7) :
         # This code checks a crash solved in Berkeley DB 4.7
diff --git a/lib-python/2.7/bsddb/test/test_thread.py b/lib-python/2.7/bsddb/test/test_thread.py
--- a/lib-python/2.7/bsddb/test/test_thread.py
+++ b/lib-python/2.7/bsddb/test/test_thread.py
@@ -85,7 +85,7 @@
         readers_per_writer=self.readers//self.writers
         self.assertEqual(self.records,self.writers*records_per_writer)
         self.assertEqual(self.readers,self.writers*readers_per_writer)
-        self.assertTrue((records_per_writer%readers_per_writer)==0)
+        self.assertEqual(records_per_writer%readers_per_writer, 0)
         readers = []
 
         for x in xrange(self.readers):
@@ -213,7 +213,7 @@
         readers_per_writer=self.readers//self.writers
         self.assertEqual(self.records,self.writers*records_per_writer)
         self.assertEqual(self.readers,self.writers*readers_per_writer)
-        self.assertTrue((records_per_writer%readers_per_writer)==0)
+        self.assertEqual(records_per_writer%readers_per_writer, 0)
 
         readers = []
         for x in xrange(self.readers):
@@ -339,7 +339,7 @@
         readers_per_writer=self.readers//self.writers
         self.assertEqual(self.records,self.writers*records_per_writer)
         self.assertEqual(self.readers,self.writers*readers_per_writer)
-        self.assertTrue((records_per_writer%readers_per_writer)==0)
+        self.assertEqual(records_per_writer%readers_per_writer, 0)
 
         readers=[]
         for x in xrange(self.readers):
diff --git a/lib-python/2.7/cProfile.py b/lib-python/2.7/cProfile.py
--- a/lib-python/2.7/cProfile.py
+++ b/lib-python/2.7/cProfile.py
@@ -64,11 +64,11 @@
 # ____________________________________________________________
 
 class Profile(_lsprof.Profiler):
-    """Profile(custom_timer=None, time_unit=None, subcalls=True, builtins=True)
+    """Profile(timer=None, timeunit=None, subcalls=True, builtins=True)
 
     Builds a profiler object using the specified timer function.
     The default timer is a fast built-in one based on real time.
-    For custom timer functions returning integers, time_unit can
+    For custom timer functions returning integers, timeunit can
     be a float specifying a scale (i.e. how long each integer unit
     is, in seconds).
     """
@@ -161,7 +161,7 @@
 # ____________________________________________________________
 
 def main():
-    import os, sys, types
+    import os, sys, pstats
     from optparse import OptionParser
     usage = "cProfile.py [-o output_file_path] [-s sort] scriptfile [arg] ..."
     parser = OptionParser(usage=usage)
@@ -170,7 +170,8 @@
         help="Save stats to <outfile>", default=None)
     parser.add_option('-s', '--sort', dest="sort",
         help="Sort order when printing to stdout, based on pstats.Stats class",
-        default=-1)
+        default=-1,
+        choices=sorted(pstats.Stats.sort_arg_dict_default))
 
     if not sys.argv[1:]:
         parser.print_usage()
@@ -184,10 +185,12 @@
         sys.path.insert(0, os.path.dirname(progname))
         with open(progname, 'rb') as fp:
             code = compile(fp.read(), progname, 'exec')
-        mainmod = types.ModuleType('__main__')
-        mainmod.__file__ = progname
-        mainmod.__package__ = None
-        runctx(code, mainmod.__dict__, None, options.outfile, options.sort)
+        globs = {
+            '__file__': progname,
+            '__name__': '__main__',
+            '__package__': None,
+        }
+        runctx(code, globs, None, options.outfile, options.sort)
     else:
         parser.print_usage()
     return parser
diff --git a/lib-python/2.7/cgi.py b/lib-python/2.7/cgi.py
--- a/lib-python/2.7/cgi.py
+++ b/lib-python/2.7/cgi.py
@@ -184,11 +184,12 @@
     return urlparse.parse_qs(qs, keep_blank_values, strict_parsing)
 
 
-def parse_qsl(qs, keep_blank_values=0, strict_parsing=0):
+def parse_qsl(qs, keep_blank_values=0, strict_parsing=0, max_num_fields=None):
     """Parse a query given as a string argument."""
     warn("cgi.parse_qsl is deprecated, use urlparse.parse_qsl instead",
          PendingDeprecationWarning, 2)
-    return urlparse.parse_qsl(qs, keep_blank_values, strict_parsing)
+    return urlparse.parse_qsl(qs, keep_blank_values, strict_parsing,
+                              max_num_fields)
 
 def parse_multipart(fp, pdict):
     """Parse multipart input.
@@ -393,7 +394,8 @@
     """
 
     def __init__(self, fp=None, headers=None, outerboundary="",
-                 environ=os.environ, keep_blank_values=0, strict_parsing=0):
+                 environ=os.environ, keep_blank_values=0, strict_parsing=0,
+                 max_num_fields=None):
         """Constructor.  Read multipart/* until last part.
 
         Arguments, all optional:
@@ -420,10 +422,14 @@
             If false (the default), errors are silently ignored.
             If true, errors raise a ValueError exception.
 
+        max_num_fields: int. If set, then __init__ throws a ValueError
+            if there are more than n fields read by parse_qsl().
+
         """
         method = 'GET'
         self.keep_blank_values = keep_blank_values
         self.strict_parsing = strict_parsing
+        self.max_num_fields = max_num_fields
         if 'REQUEST_METHOD' in environ:
             method = environ['REQUEST_METHOD'].upper()
         self.qs_on_post = None
@@ -606,10 +612,9 @@
         qs = self.fp.read(self.length)
         if self.qs_on_post:
             qs += '&' + self.qs_on_post
-        self.list = list = []
-        for key, value in urlparse.parse_qsl(qs, self.keep_blank_values,
-                                            self.strict_parsing):
-            list.append(MiniFieldStorage(key, value))
+        query = urlparse.parse_qsl(qs, self.keep_blank_values,
+                                   self.strict_parsing, self.max_num_fields)
+        self.list = [MiniFieldStorage(key, value) for key, value in query]
         self.skip_lines()
 
     FieldStorageClass = None
@@ -621,19 +626,38 @@
             raise ValueError, 'Invalid boundary in multipart form: %r' % (ib,)
         self.list = []
         if self.qs_on_post:
-            for key, value in urlparse.parse_qsl(self.qs_on_post,
-                                self.keep_blank_values, self.strict_parsing):
-                self.list.append(MiniFieldStorage(key, value))
+            query = urlparse.parse_qsl(self.qs_on_post,
+                                       self.keep_blank_values,
+                                       self.strict_parsing,
+                                       self.max_num_fields)
+            self.list.extend(MiniFieldStorage(key, value)
+                             for key, value in query)
             FieldStorageClass = None
 
+        # Propagate max_num_fields into the sub class appropriately
+        max_num_fields = self.max_num_fields
+        if max_num_fields is not None:
+            max_num_fields -= len(self.list)
+
         klass = self.FieldStorageClass or self.__class__
         part = klass(self.fp, {}, ib,
-                     environ, keep_blank_values, strict_parsing)
+                     environ, keep_blank_values, strict_parsing,
+                     max_num_fields)
+
         # Throw first part away
         while not part.done:
             headers = rfc822.Message(self.fp)
             part = klass(self.fp, headers, ib,
-                         environ, keep_blank_values, strict_parsing)
+                         environ, keep_blank_values, strict_parsing,
+                         max_num_fields)
+
+            if max_num_fields is not None:
+                max_num_fields -= 1
+                if part.list:
+                    max_num_fields -= len(part.list)
+                if max_num_fields < 0:
+                    raise ValueError('Max number of fields exceeded')
+
             self.list.append(part)
         self.skip_lines()
 
diff --git a/lib-python/2.7/cgitb.py b/lib-python/2.7/cgitb.py
--- a/lib-python/2.7/cgitb.py
+++ b/lib-python/2.7/cgitb.py
@@ -125,7 +125,7 @@
         args, varargs, varkw, locals = inspect.getargvalues(frame)
         call = ''
         if func != '?':
-            call = 'in ' + strong(func) + \
+            call = 'in ' + strong(pydoc.html.escape(func)) + \
                 inspect.formatargvalues(args, varargs, varkw, locals,
                     formatvalue=lambda value: '=' + pydoc.html.repr(value))
 
@@ -285,7 +285,7 @@
 
         if self.display:
             if plain:
-                doc = doc.replace('&', '&amp;').replace('<', '&lt;')
+                doc = pydoc.html.escape(doc)
                 self.file.write('<pre>' + doc + '</pre>\n')
             else:
                 self.file.write(doc + '\n')
diff --git a/lib-python/2.7/code.py b/lib-python/2.7/code.py
--- a/lib-python/2.7/code.py
+++ b/lib-python/2.7/code.py
@@ -104,12 +104,6 @@
         except SystemExit:
             raise
         except:
-            if softspace(sys.stdout, 0):
-                print
-            try:
-                sys.stdout.flush()
-            except:
-                pass
             self.showtraceback()
         else:
             if softspace(sys.stdout, 0):
diff --git a/lib-python/2.7/codecs.py b/lib-python/2.7/codecs.py
--- a/lib-python/2.7/codecs.py
+++ b/lib-python/2.7/codecs.py
@@ -472,15 +472,17 @@
             self.charbuffer = "".join(self.linebuffer)
             self.linebuffer = None
 
+        if chars < 0:
+            # For compatibility with other read() methods that take a
+            # single argument
+            chars = size
+
         # read until we get the required number of characters (if available)
         while True:
             # can the request be satisfied from the character buffer?
             if chars >= 0:
                 if len(self.charbuffer) >= chars:
                     break
-            elif size >= 0:
-                if len(self.charbuffer) >= size:
-                    break
             # we need more data
             if size < 0:
                 newdata = self.stream.read()
diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py
--- a/lib-python/2.7/collections.py
+++ b/lib-python/2.7/collections.py
@@ -24,19 +24,6 @@
 import heapq as _heapq
 from itertools import repeat as _repeat, chain as _chain, starmap as _starmap
 from itertools import imap as _imap
-try:
-    from __pypy__ import newdict
-except ImportError:
-    assert '__pypy__' not in _sys.builtin_module_names
-    newdict = lambda _ : {}
-try:
-    from __pypy__ import reversed_dict as _reversed_dict
-except ImportError:
-    _reversed_dict = None     # don't have ordered dicts
-try:
-    from __pypy__ import dict_popitem_first as _dict_popitem_first
-except ImportError:
-    _dict_popitem_first = None
 
 try:
     from thread import get_ident as _get_ident
@@ -48,104 +35,8 @@
 ### OrderedDict
 ################################################################################
 
-if _dict_popitem_first is None:
-    def _dict_popitem_first(self):
-        it = dict.iteritems(self)
-        try:
-            k, v = it.next()
-        except StopIteration:
-            raise KeyError('dictionary is empty')
-        dict.__delitem__(self, k)
-        return (k, v)
-
-
 class OrderedDict(dict):
-    '''Dictionary that remembers insertion order.
-
-    In PyPy all dicts are ordered anyway.  This is mostly useful as a
-    placeholder to mean "this dict must be ordered even on CPython".
-
-    Known difference: iterating over an OrderedDict which is being
-    concurrently modified raises RuntimeError in PyPy.  In CPython
-    instead we get some behavior that appears reasonable in some
-    cases but is nonsensical in other cases.  This is officially
-    forbidden by the CPython docs, so we forbid it explicitly for now.
-    '''
-
-    def __reversed__(self):
-        return _reversed_dict(self)
-
-    def popitem(self, last=True):
-        '''od.popitem() -> (k, v), return and remove a (key, value) pair.
-        Pairs are returned in LIFO order if last is true or FIFO order if false.
-
-        '''
-        if last:
-            return dict.popitem(self)
-        else:
-            return _dict_popitem_first(self)
-
-    def __repr__(self, _repr_running={}):
-        'od.__repr__() <==> repr(od)'
-        call_key = id(self), _get_ident()
-        if call_key in _repr_running:
-            return '...'
-        _repr_running[call_key] = 1
-        try:
-            if not self:
-                return '%s()' % (self.__class__.__name__,)
-            return '%s(%r)' % (self.__class__.__name__, self.items())
-        finally:
-            del _repr_running[call_key]
-
-    def __reduce__(self):
-        'Return state information for pickling'
-        items = [[k, self[k]] for k in self]
-        inst_dict = vars(self).copy()
-        if inst_dict:
-            return (self.__class__, (items,), inst_dict)
-        return self.__class__, (items,)
-
-    def copy(self):
-        'od.copy() -> a shallow copy of od'
-        return self.__class__(self)
-
-    def __eq__(self, other):
-        '''od.__eq__(y) <==> od==y.  Comparison to another OD is order-sensitive
-        while comparison to a regular mapping is order-insensitive.
-
-        '''
-        if isinstance(other, OrderedDict):
-            return dict.__eq__(self, other) and all(_imap(_eq, self, other))
-        return dict.__eq__(self, other)
-
-    def __ne__(self, other):
-        'od.__ne__(y) <==> od!=y'
-        return not self == other
-
-    # -- the following methods support python 3.x style dictionary views --
-
-    def viewkeys(self):
-        "od.viewkeys() -> a set-like object providing a view on od's keys"
-        return KeysView(self)
-
-    def viewvalues(self):
-        "od.viewvalues() -> an object providing a view on od's values"
-        return ValuesView(self)
-
-    def viewitems(self):
-        "od.viewitems() -> a set-like object providing a view on od's items"
-        return ItemsView(self)
-
-
-def _compat_with_unordered_dicts():
-    # This returns the methods needed in OrderedDict in case the base
-    # 'dict' class is not actually ordered, like on top of CPython or
-    # old PyPy or PyPy-STM.
-
-    # ===== Original comments and code follows      =====
-    # ===== The unmodified methods are not repeated =====
-
+    'Dictionary that remembers insertion order'
     # An inherited dict maps keys to values.
     # The inherited dict provides __getitem__, __len__, __contains__, and get.
     # The remaining methods are order-aware.
@@ -156,12 +47,17 @@
     # The sentinel element never gets deleted (this simplifies the algorithm).
     # Each link is stored as a list of length three:  [PREV, NEXT, KEY].
 
-    def __init__(self, *args, **kwds):
+    def __init__(*args, **kwds):
         '''Initialize an ordered dictionary.  The signature is the same as
         regular dictionaries, but keyword arguments are not recommended because
         their insertion order is arbitrary.
 
         '''
+        if not args:
+            raise TypeError("descriptor '__init__' of 'OrderedDict' object "
+                            "needs an argument")
+        self = args[0]
+        args = args[1:]
         if len(args) > 1:
             raise TypeError('expected at most 1 arguments, got %d' % len(args))
         try:
@@ -282,6 +178,19 @@
         value = self.pop(key)
         return key, value
 
+    def __repr__(self, _repr_running={}):
+        'od.__repr__() <==> repr(od)'
+        call_key = id(self), _get_ident()
+        if call_key in _repr_running:
+            return '...'
+        _repr_running[call_key] = 1
+        try:
+            if not self:
+                return '%s()' % (self.__class__.__name__,)
+            return '%s(%r)' % (self.__class__.__name__, self.items())
+        finally:
+            del _repr_running[call_key]
+
     def __reduce__(self):
         'Return state information for pickling'
         items = [[k, self[k]] for k in self]
@@ -292,6 +201,10 @@
             return (self.__class__, (items,), inst_dict)
         return self.__class__, (items,)
 
+    def copy(self):
+        'od.copy() -> a shallow copy of od'
+        return self.__class__(self)
+
     @classmethod
     def fromkeys(cls, iterable, value=None):
         '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S.
@@ -303,12 +216,33 @@
             self[key] = value
         return self
 
-    return locals()
+    def __eq__(self, other):
+        '''od.__eq__(y) <==> od==y.  Comparison to another OD is order-sensitive
+        while comparison to a regular mapping is order-insensitive.
 
-if _reversed_dict is None:
-    for _key, _value in _compat_with_unordered_dicts().items():
-        setattr(OrderedDict, _key, _value)
-    del _key, _value
+        '''
+        if isinstance(other, OrderedDict):
+            return dict.__eq__(self, other) and all(_imap(_eq, self, other))
+        return dict.__eq__(self, other)
+
+    def __ne__(self, other):
+        'od.__ne__(y) <==> od!=y'
+        return not self == other
+
+    # -- the following methods support python 3.x style dictionary views --
+
+    def viewkeys(self):
+        "od.viewkeys() -> a set-like object providing a view on od's keys"
+        return KeysView(self)
+
+    def viewvalues(self):
+        "od.viewvalues() -> an object providing a view on od's values"
+        return ValuesView(self)
+
+    def viewitems(self):
+        "od.viewitems() -> a set-like object providing a view on od's items"
+        return ItemsView(self)
+
 
 ################################################################################
 ### namedtuple
@@ -365,7 +299,7 @@
 _repr_template = '{name}=%r'
 
 _field_template = '''\
-    {name} = _property(lambda self: self[{index:d}], doc='Alias for field number {index:d}')
+    {name} = _property(_itemgetter({index:d}), doc='Alias for field number {index:d}')
 '''
 
 def namedtuple(typename, field_names, verbose=False, rename=False):
@@ -446,11 +380,8 @@
 
     # Execute the template string in a temporary namespace and support
     # tracing utilities by setting a value for frame.f_globals['__name__']
-    namespace = newdict('module')
-    namespace['__name__'] = 'namedtuple_%s' % typename
-    namespace['OrderedDict'] = OrderedDict
-    namespace['_property'] = property
-    namespace['_tuple'] = tuple
+    namespace = dict(_itemgetter=_itemgetter, __name__='namedtuple_%s' % typename,
+                     OrderedDict=OrderedDict, _property=property, _tuple=tuple)
     try:
         exec class_definition in namespace
     except SyntaxError as e:
diff --git a/lib-python/2.7/compiler/pyassem.py b/lib-python/2.7/compiler/pyassem.py
--- a/lib-python/2.7/compiler/pyassem.py
+++ b/lib-python/2.7/compiler/pyassem.py
@@ -581,7 +581,7 @@
 
 def twobyte(val):
     """Convert an int argument into high and low bytes"""
-    assert isinstance(val, int)
+    assert isinstance(val, (int, long))
     return divmod(val, 256)
 
 class LineAddrTable:
diff --git a/lib-python/2.7/compiler/transformer.py b/lib-python/2.7/compiler/transformer.py
--- a/lib-python/2.7/compiler/transformer.py
+++ b/lib-python/2.7/compiler/transformer.py
@@ -1526,7 +1526,7 @@
 def debug_tree(tree):
     l = []
     for elt in tree:
-        if isinstance(elt, int):
+        if isinstance(elt, (int, long)):
             l.append(_names.get(elt, elt))
         elif isinstance(elt, str):
             l.append(elt)
diff --git a/lib-python/2.7/copy_reg.py b/lib-python/2.7/copy_reg.py
--- a/lib-python/2.7/copy_reg.py
+++ b/lib-python/2.7/copy_reg.py
@@ -127,7 +127,11 @@
                         continue
                     # mangled names
                     elif name.startswith('__') and not name.endswith('__'):
-                        names.append('_%s%s' % (c.__name__, name))
+                        stripped = c.__name__.lstrip('_')
+                        if stripped:
+                            names.append('_%s%s' % (stripped, name))
+                        else:
+                            names.append(name)
                     else:
                         names.append(name)
 
diff --git a/lib-python/2.7/csv.py b/lib-python/2.7/csv.py
--- a/lib-python/2.7/csv.py
+++ b/lib-python/2.7/csv.py
@@ -217,7 +217,7 @@
         matches = []
         for restr in ('(?P<delim>[^\w\n"\'])(?P<space> ?)(?P<quote>["\']).*?(?P=quote)(?P=delim)', # ,".*?",
                       '(?:^|\n)(?P<quote>["\']).*?(?P=quote)(?P<delim>[^\w\n"\'])(?P<space> ?)',   #  ".*?",
-                      '(?P<delim>>[^\w\n"\'])(?P<space> ?)(?P<quote>["\']).*?(?P=quote)(?:$|\n)',  # ,".*?"
+                      '(?P<delim>[^\w\n"\'])(?P<space> ?)(?P<quote>["\']).*?(?P=quote)(?:$|\n)',   # ,".*?"
                       '(?:^|\n)(?P<quote>["\']).*?(?P=quote)(?:$|\n)'):                            #  ".*?" (no delim, no space)
             regexp = re.compile(restr, re.DOTALL | re.MULTILINE)
             matches = regexp.findall(data)
diff --git a/lib-python/2.7/ctypes/__init__.py b/lib-python/2.7/ctypes/__init__.py
--- a/lib-python/2.7/ctypes/__init__.py
+++ b/lib-python/2.7/ctypes/__init__.py
@@ -4,7 +4,6 @@
 
 __version__ = "1.1.0"
 
-import _ffi
 from _ctypes import Union, Structure, Array
 from _ctypes import _Pointer
 from _ctypes import CFuncPtr as _CFuncPtr
@@ -343,6 +342,10 @@
     """
     _func_flags_ = _FUNCFLAG_CDECL
     _func_restype_ = c_int
+    # default values for repr
+    _name = '<uninitialized>'
+    _handle = 0
+    _FuncPtr = None
 
     def __init__(self, name, mode=DEFAULT_MODE, handle=None,
                  use_errno=False,
@@ -360,16 +363,9 @@
         self._FuncPtr = _FuncPtr
 
         if handle is None:
-            handle = 0
-        if flags & _FUNCFLAG_CDECL:
-            pypy_dll = _ffi.CDLL(name, mode, handle)
+            self._handle = _dlopen(self._name, mode)
         else:
-            pypy_dll = _ffi.WinDLL(name, mode, handle)
-        self.__pypy_dll__ = pypy_dll
-        handle = int(pypy_dll)
-        if _sys.maxint > 2 ** 32:
-            handle = int(handle)   # long -> int
-        self._handle = handle
+            self._handle = handle
 
     def __repr__(self):
         return "<%s '%s', handle %x at %x>" % \
@@ -390,13 +386,12 @@
             func.__name__ = name_or_ordinal
         return func
 
-# Not in PyPy
-#class PyDLL(CDLL):
-#    """This class represents the Python library itself.  It allows
-#    accessing Python API functions.  The GIL is not released, and
-#    Python exceptions are handled correctly.
-#    """
-#    _func_flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI
+class PyDLL(CDLL):
+    """This class represents the Python library itself.  It allows
+    accessing Python API functions.  The GIL is not released, and
+    Python exceptions are handled correctly.
+    """
+    _func_flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI
 
 if _os.name in ("nt", "ce"):
 
@@ -449,8 +444,15 @@
         return self._dlltype(name)
 
 cdll = LibraryLoader(CDLL)
-# not on PyPy
-#pydll = LibraryLoader(PyDLL)
+pydll = LibraryLoader(PyDLL)
+
+if _os.name in ("nt", "ce"):
+    pythonapi = PyDLL("python dll", None, _sys.dllhandle)
+elif _sys.platform == "cygwin":
+    pythonapi = PyDLL("libpython%d.%d.dll" % _sys.version_info[:2])
+else:
+    pythonapi = PyDLL(None)
+
 
 if _os.name in ("nt", "ce"):
     windll = LibraryLoader(WinDLL)
@@ -496,12 +498,9 @@
         _flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI
     return CFunctionType
 
+_cast = PYFUNCTYPE(py_object, c_void_p, py_object, py_object)(_cast_addr)
 def cast(obj, typ):
-    try:
-        c_void_p.from_param(obj)
-    except TypeError, e:
-        raise ArgumentError(str(e))
-    return _cast_addr(obj, obj, typ)
+    return _cast(obj, obj, typ)
 
 _string_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr)
 def string_at(ptr, size=-1):
diff --git a/lib-python/2.7/ctypes/test/__init__.py b/lib-python/2.7/ctypes/test/__init__.py
--- a/lib-python/2.7/ctypes/test/__init__.py
+++ b/lib-python/2.7/ctypes/test/__init__.py
@@ -214,16 +214,3 @@
         result = unittest.TestResult()
         test(result)
         return result
-
-def xfail(method):
-    """
-    Poor's man xfail: remove it when all the failures have been fixed
-    """
-    def new_method(self, *args, **kwds):
-        try:
-            method(self, *args, **kwds)
-        except:
-            pass
-        else:
-            self.assertTrue(False, "DID NOT RAISE")
-    return new_method
diff --git a/lib-python/2.7/ctypes/test/test_anon.py b/lib-python/2.7/ctypes/test/test_anon.py
--- a/lib-python/2.7/ctypes/test/test_anon.py
+++ b/lib-python/2.7/ctypes/test/test_anon.py
@@ -1,4 +1,5 @@
 import unittest
+from test.support import cpython_only
 from ctypes import *
 
 class AnonTest(unittest.TestCase):
@@ -35,6 +36,18 @@
                                                       {"_fields_": [],
                                                        "_anonymous_": ["x"]}))
 
+    @cpython_only
+    def test_issue31490(self):
+        # There shouldn't be an assertion failure in case the class has an
+        # attribute whose name is specified in _anonymous_ but not in _fields_.
+
+        # AttributeError: 'x' is specified in _anonymous_ but not in _fields_
+        with self.assertRaises(AttributeError):
+            class Name(Structure):
+                _fields_ = []
+                _anonymous_ = ["x"]
+                x = 42
+
     def test_nested(self):
         class ANON_S(Structure):
             _fields_ = [("a", c_int)]
diff --git a/lib-python/2.7/ctypes/test/test_arrays.py b/lib-python/2.7/ctypes/test/test_arrays.py
--- a/lib-python/2.7/ctypes/test/test_arrays.py
+++ b/lib-python/2.7/ctypes/test/test_arrays.py
@@ -1,25 +1,16 @@
 import unittest
+from test.support import precisionbigmemtest, _2G
+import sys
 from ctypes import *
-from test.test_support import impl_detail
 
 from ctypes.test import need_symbol
 
 formats = "bBhHiIlLqQfd"
 
-# c_longdouble commented out for PyPy, look at the commend in test_longdouble
 formats = c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, \
-          c_long, c_ulonglong, c_float, c_double #, c_longdouble
+          c_long, c_ulonglong, c_float, c_double, c_longdouble
 
 class ArrayTestCase(unittest.TestCase):
-
-    @impl_detail('long double not supported by PyPy', pypy=False)
-    def test_longdouble(self):
-        """
-        This test is empty. It's just here to remind that we commented out
-        c_longdouble in "formats". If pypy will ever supports c_longdouble, we
-        should kill this test and uncomment c_longdouble inside formats.
-        """
-
     def test_simple(self):
         # create classes holding simple numeric types, and check
         # various properties.
@@ -143,5 +134,10 @@
         t2 = my_int * 1
         self.assertIs(t1, t2)
 
+    @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform')
+    @precisionbigmemtest(size=_2G, memuse=1, dry_run=False)
+    def test_large_array(self, size):
+        a = c_char * size
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_as_parameter.py b/lib-python/2.7/ctypes/test/test_as_parameter.py
--- a/lib-python/2.7/ctypes/test/test_as_parameter.py
+++ b/lib-python/2.7/ctypes/test/test_as_parameter.py
@@ -24,7 +24,7 @@
         f.argtypes = [c_byte, c_wchar, c_int, c_long, c_float, c_double]
         result = f(self.wrap(1), self.wrap(u"x"), self.wrap(3), self.wrap(4), self.wrap(5.0), self.wrap(6.0))
         self.assertEqual(result, 139)
-        self.assertTrue(type(result), int)
+        self.assertIs(type(result), int)
 
     def test_pointers(self):
         f = dll._testfunc_p_p
diff --git a/lib-python/2.7/ctypes/test/test_bitfields.py b/lib-python/2.7/ctypes/test/test_bitfields.py
--- a/lib-python/2.7/ctypes/test/test_bitfields.py
+++ b/lib-python/2.7/ctypes/test/test_bitfields.py
@@ -1,8 +1,7 @@
 from ctypes import *
-from ctypes.test import need_symbol, xfail
+from ctypes.test import need_symbol
 import unittest
 import os
-import sys
 
 import ctypes
 import _ctypes_test
@@ -117,28 +116,23 @@
     def test_nonint_types(self):
         # bit fields are not allowed on non-integer types.
         result = self.fail_fields(("a", c_char_p, 1))
-        self.assertEqual(result[0], TypeError)
-        self.assertIn('bit fields not allowed for type', result[1])
+        self.assertEqual(result, (TypeError, 'bit fields not allowed for type c_char_p'))
 
         result = self.fail_fields(("a", c_void_p, 1))
-        self.assertEqual(result[0], TypeError)
-        self.assertIn('bit fields not allowed for type', result[1])
+        self.assertEqual(result, (TypeError, 'bit fields not allowed for type c_void_p'))
 
         if c_int != c_long:
             result = self.fail_fields(("a", POINTER(c_int), 1))
-            self.assertEqual(result[0], TypeError)
-            self.assertIn('bit fields not allowed for type', result[1])
+            self.assertEqual(result, (TypeError, 'bit fields not allowed for type LP_c_int'))
 
         result = self.fail_fields(("a", c_char, 1))
-        self.assertEqual(result[0], TypeError)
-        self.assertIn('bit fields not allowed for type', result[1])
+        self.assertEqual(result, (TypeError, 'bit fields not allowed for type c_char'))
 
         class Dummy(Structure):
             _fields_ = []
 
         result = self.fail_fields(("a", Dummy, 1))
-        self.assertEqual(result[0], TypeError)
-        self.assertIn('bit fields not allowed for type', result[1])
+        self.assertEqual(result, (TypeError, 'bit fields not allowed for type Dummy'))
 
     @need_symbol('c_wchar')
     def test_c_wchar(self):
@@ -272,13 +266,12 @@
             _fields_ = [("a", c_uint32, 24),
                         ("b", c_uint32, 4),
                         ("c", c_uint32, 4)]
-        import array
-        b = array.array("c", '\x00' * 4)
+        b = bytearray(4)
         x = Little.from_buffer(b)
         x.a = 0xabcdef
         x.b = 1
         x.c = 2
-        self.assertEqual(b.tostring(), b'\xef\xcd\xab\x21')
+        self.assertEqual(b, b'\xef\xcd\xab\x21')
 
     @need_symbol('c_uint32')
     def test_uint32_swap_big_endian(self):
@@ -287,19 +280,12 @@
             _fields_ = [("a", c_uint32, 24),
                         ("b", c_uint32, 4),
                         ("c", c_uint32, 4)]
-        import array
-        b = array.array("c", '\x00' * 4)
+        b = bytearray(4)
         x = Big.from_buffer(b)
         x.a = 0xabcdef
         x.b = 1
         x.c = 2
-        self.assertEqual(b.tostring(), b'\xab\xcd\xef\x12')
-
-    # see issue #1213, on big endian it fails for the little endian case
-    if sys.byteorder == 'little':
-        test_uint32_swap_big_endian = xfail(test_uint32_swap_big_endian)
-    elif sys.byteorder == 'big':
-        test_uint32_swap_little_endian = xfail(test_uint32_swap_little_endian)
+        self.assertEqual(b, b'\xab\xcd\xef\x12')
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_byteswap.py b/lib-python/2.7/ctypes/test/test_byteswap.py
--- a/lib-python/2.7/ctypes/test/test_byteswap.py
+++ b/lib-python/2.7/ctypes/test/test_byteswap.py
@@ -2,7 +2,6 @@
 from binascii import hexlify
 
 from ctypes import *
-from ctypes.test import xfail
 
 def bin(s):
     return hexlify(memoryview(s)).upper()
diff --git a/lib-python/2.7/ctypes/test/test_callbacks.py b/lib-python/2.7/ctypes/test/test_callbacks.py
--- a/lib-python/2.7/ctypes/test/test_callbacks.py
+++ b/lib-python/2.7/ctypes/test/test_callbacks.py
@@ -2,7 +2,6 @@
 import unittest
 from ctypes import *
 from ctypes.test import need_symbol
-from ctypes.test import xfail
 import _ctypes_test
 
 class Callbacks(unittest.TestCase):
@@ -102,7 +101,6 @@
         self.check_type(c_char_p, "abc")
         self.check_type(c_char_p, "def")
 
-    @xfail
     def test_pyobject(self):
         o = ()
         from sys import getrefcount as grc
@@ -252,6 +250,7 @@
     def test_callback_large_struct(self):
         class Check: pass
 
+        # This should mirror the structure in Modules/_ctypes/_ctypes_test.c
         class X(Structure):
             _fields_ = [
                 ('first', c_ulong),
@@ -263,6 +262,11 @@
             check.first = s.first
             check.second = s.second
             check.third = s.third
+            # See issue #29565.
+            # The structure should be passed by value, so
+            # any changes to it should not be reflected in
+            # the value passed
+            s.first = s.second = s.third = 0x0badf00d
 
         check = Check()
         s = X()
@@ -283,6 +287,11 @@
         self.assertEqual(check.first, 0xdeadbeef)
         self.assertEqual(check.second, 0xcafebabe)
         self.assertEqual(check.third, 0x0bad1dea)
+        # See issue #29565.
+        # Ensure that the original struct is unchanged.
+        self.assertEqual(s.first, check.first)
+        self.assertEqual(s.second, check.second)
+        self.assertEqual(s.third, check.third)
 
 ################################################################
 
diff --git a/lib-python/2.7/ctypes/test/test_cfuncs.py b/lib-python/2.7/ctypes/test/test_cfuncs.py
--- a/lib-python/2.7/ctypes/test/test_cfuncs.py
+++ b/lib-python/2.7/ctypes/test/test_cfuncs.py
@@ -6,7 +6,6 @@
 from ctypes.test import need_symbol
 
 import _ctypes_test
-from test.test_support import impl_detail
 
 class CFunctions(unittest.TestCase):
     _dll = CDLL(_ctypes_test.__file__)
@@ -160,14 +159,12 @@
         self.assertEqual(self._dll.tf_bd(0, 42.), 14.)
         self.assertEqual(self.S(), 42)
 
-    @impl_detail('long double not supported by PyPy', pypy=False)
     def test_longdouble(self):
         self._dll.tf_D.restype = c_longdouble
         self._dll.tf_D.argtypes = (c_longdouble,)
         self.assertEqual(self._dll.tf_D(42.), 14.)
         self.assertEqual(self.S(), 42)
 
-    @impl_detail('long double not supported by PyPy', pypy=False)
     def test_longdouble_plus(self):
         self._dll.tf_bD.restype = c_longdouble
         self._dll.tf_bD.argtypes = (c_byte, c_longdouble)
diff --git a/lib-python/2.7/ctypes/test/test_delattr.py b/lib-python/2.7/ctypes/test/test_delattr.py
--- a/lib-python/2.7/ctypes/test/test_delattr.py
+++ b/lib-python/2.7/ctypes/test/test_delattr.py
@@ -6,15 +6,15 @@
 
 class TestCase(unittest.TestCase):
     def test_simple(self):
-        self.assertRaises((TypeError, AttributeError),
+        self.assertRaises(TypeError,
                           delattr, c_int(42), "value")
 
     def test_chararray(self):
-        self.assertRaises((TypeError, AttributeError),
+        self.assertRaises(TypeError,
                           delattr, (c_char * 5)(), "value")
 
     def test_struct(self):
-        self.assertRaises((TypeError, AttributeError),
+        self.assertRaises(TypeError,
                           delattr, X(), "foo")
 
 if __name__ == "__main__":
diff --git a/lib-python/2.7/ctypes/test/test_frombuffer.py b/lib-python/2.7/ctypes/test/test_frombuffer.py
--- a/lib-python/2.7/ctypes/test/test_frombuffer.py
+++ b/lib-python/2.7/ctypes/test/test_frombuffer.py
@@ -78,12 +78,21 @@
                           (c_int * 1).from_buffer_copy, a, 16 * sizeof(c_int))
 
     def test_abstract(self):
+        from ctypes import _Pointer, _SimpleCData, _CFuncPtr
+
         self.assertRaises(TypeError, Array.from_buffer, bytearray(10))
         self.assertRaises(TypeError, Structure.from_buffer, bytearray(10))
         self.assertRaises(TypeError, Union.from_buffer, bytearray(10))
+        self.assertRaises(TypeError, _CFuncPtr.from_buffer, bytearray(10))
+        self.assertRaises(TypeError, _Pointer.from_buffer, bytearray(10))
+        self.assertRaises(TypeError, _SimpleCData.from_buffer, bytearray(10))
+
         self.assertRaises(TypeError, Array.from_buffer_copy, b"123")
         self.assertRaises(TypeError, Structure.from_buffer_copy, b"123")
         self.assertRaises(TypeError, Union.from_buffer_copy, b"123")
+        self.assertRaises(TypeError, _CFuncPtr.from_buffer_copy, b"123")
+        self.assertRaises(TypeError, _Pointer.from_buffer_copy, b"123")
+        self.assertRaises(TypeError, _SimpleCData.from_buffer_copy, b"123")
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_funcptr.py b/lib-python/2.7/ctypes/test/test_funcptr.py
--- a/lib-python/2.7/ctypes/test/test_funcptr.py
+++ b/lib-python/2.7/ctypes/test/test_funcptr.py
@@ -123,5 +123,10 @@
         self.assertEqual(strtok(None, "\n"), "c")
         self.assertEqual(strtok(None, "\n"), None)
 
+    def test_abstract(self):
+        from ctypes import _CFuncPtr
+
+        self.assertRaises(TypeError, _CFuncPtr, 13, "name", 42, "iid")
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_functions.py b/lib-python/2.7/ctypes/test/test_functions.py
--- a/lib-python/2.7/ctypes/test/test_functions.py
+++ b/lib-python/2.7/ctypes/test/test_functions.py
@@ -8,8 +8,6 @@
 from ctypes import *
 from ctypes.test import need_symbol
 import sys, unittest
-from ctypes.test import xfail
-from test.test_support import impl_detail
 
 try:
     WINFUNCTYPE
@@ -140,7 +138,6 @@
         self.assertEqual(result, -21)
         self.assertEqual(type(result), float)
 
-    @impl_detail('long double not supported by PyPy', pypy=False)
     def test_longdoubleresult(self):
         f = dll._testfunc_D_bhilfD
         f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_longdouble]
@@ -391,7 +388,6 @@
                 (s8i.a, s8i.b, s8i.c, s8i.d, s8i.e, s8i.f, s8i.g, s8i.h),
                 (9*2, 8*3, 7*4, 6*5, 5*6, 4*7, 3*8, 2*9))
 
-    @xfail
     def test_sf1651235(self):
         # see http://www.python.org/sf/1651235
 
diff --git a/lib-python/2.7/ctypes/test/test_internals.py b/lib-python/2.7/ctypes/test/test_internals.py
--- a/lib-python/2.7/ctypes/test/test_internals.py
+++ b/lib-python/2.7/ctypes/test/test_internals.py
@@ -1,10 +1,7 @@
 # This tests the internal _objects attribute
 import unittest
 from ctypes import *
-try:
-    from sys import getrefcount as grc
-except ImportError:
-    grc = None      # e.g. PyPy
+from sys import getrefcount as grc
 
 # XXX This test must be reviewed for correctness!!!
 
@@ -25,8 +22,6 @@
         self.assertEqual(id(a), id(b))
 
     def test_ints(self):
-        if grc is None:
-            return unittest.skip("no sys.getrefcount()")
         i = 42000123
         refcnt = grc(i)
         ci = c_int(i)
@@ -34,19 +29,11 @@
         self.assertEqual(ci._objects, None)
 
     def test_c_char_p(self):
-        if grc is None:
-            return unittest.skip("no sys.getrefcount()")
         s = "Hello, World"
         refcnt = grc(s)
         cs = c_char_p(s)
         self.assertEqual(refcnt + 1, grc(s))
-        try:
-            # Moving gcs need to allocate a nonmoving buffer
-            cs._objects._obj
-        except AttributeError:
-            self.assertSame(cs._objects, s)
-        else:
-            self.assertSame(cs._objects._obj, s)
+        self.assertSame(cs._objects, s)
 
     def test_simple_struct(self):
         class X(Structure):
diff --git a/lib-python/2.7/ctypes/test/test_libc.py b/lib-python/2.7/ctypes/test/test_libc.py
--- a/lib-python/2.7/ctypes/test/test_libc.py
+++ b/lib-python/2.7/ctypes/test/test_libc.py
@@ -25,14 +25,5 @@
         lib.my_qsort(chars, len(chars)-1, sizeof(c_char), comparefunc(sort))
         self.assertEqual(chars.raw, "   ,,aaaadmmmnpppsss\x00")
 
-    def SKIPPED_test_no_more_xfail(self):
-        # We decided to not explicitly support the whole ctypes-2.7
-        # and instead go for a case-by-case, demand-driven approach.
-        # So this test is skipped instead of failing.
-        import socket
-        import ctypes.test
-        self.assertTrue(not hasattr(ctypes.test, 'xfail'),
-                        "You should incrementally grep for '@xfail' and remove them, they are real failures")
-
 if __name__ == "__main__":
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_loading.py b/lib-python/2.7/ctypes/test/test_loading.py
--- a/lib-python/2.7/ctypes/test/test_loading.py
+++ b/lib-python/2.7/ctypes/test/test_loading.py
@@ -2,7 +2,8 @@
 import sys, unittest
 import os
 from ctypes.util import find_library
-from ctypes.test import is_resource_enabled, xfail
+from ctypes.test import is_resource_enabled
+import test.test_support as support
 
 libc_name = None
 if os.name == "nt":
@@ -27,6 +28,12 @@
         CDLL(os.path.basename(libc_name))
         self.assertRaises(OSError, CDLL, self.unknowndll)
 
+    @support.requires_unicode
+    @unittest.skipUnless(libc_name is not None, 'could not find libc')
+    def test_load_unicode(self):
+        CDLL(unicode(libc_name))
+        self.assertRaises(OSError, CDLL, unicode(self.unknowndll))
+
     @unittest.skipUnless(libc_name is not None, 'could not find libc')
     @unittest.skipUnless(libc_name is not None and
                          os.path.basename(libc_name) == "libc.so.6",
@@ -80,7 +87,6 @@
 
         self.assertRaises(AttributeError, dll.__getitem__, 1234)
 
-    @xfail
     @unittest.skipUnless(os.name == "nt", 'Windows-specific test')
     def test_1703286_A(self):
         from _ctypes import LoadLibrary, FreeLibrary
@@ -92,7 +98,6 @@
         handle = LoadLibrary("advapi32")
         FreeLibrary(handle)
 
-    @xfail
     @unittest.skipUnless(os.name == "nt", 'Windows-specific test')
     def test_1703286_B(self):
         # Since on winXP 64-bit advapi32 loads like described
diff --git a/lib-python/2.7/ctypes/test/test_memfunctions.py b/lib-python/2.7/ctypes/test/test_memfunctions.py
--- a/lib-python/2.7/ctypes/test/test_memfunctions.py
+++ b/lib-python/2.7/ctypes/test/test_memfunctions.py
@@ -55,8 +55,7 @@
         s = string_at("foo bar")
         # XXX The following may be wrong, depending on how Python
         # manages string instances
-        if hasattr(sys, 'getrefcount'):
-            self.assertEqual(2, sys.getrefcount(s))
+        self.assertEqual(2, sys.getrefcount(s))
         self.assertTrue(s, "foo bar")
 
         self.assertEqual(string_at("foo bar", 8), "foo bar\0")
diff --git a/lib-python/2.7/ctypes/test/test_numbers.py b/lib-python/2.7/ctypes/test/test_numbers.py
--- a/lib-python/2.7/ctypes/test/test_numbers.py
+++ b/lib-python/2.7/ctypes/test/test_numbers.py
@@ -1,7 +1,6 @@
 from ctypes import *
 import unittest
 import struct
-from ctypes.test import xfail
 
 def valid_ranges(*types):
     # given a sequence of numeric types, collect their _type_
@@ -91,14 +90,12 @@
             self.assertRaises(ValueError, t, l-1)
             self.assertRaises(ValueError, t, h+1)
 
-    @xfail
     def test_from_param(self):
         # the from_param class method attribute always
         # returns PyCArgObject instances
         for t in signed_types + unsigned_types + float_types:
             self.assertEqual(ArgType, type(t.from_param(0)))
 
-    @xfail
     def test_byref(self):
         # calling byref returns also a PyCArgObject instance
         for t in signed_types + unsigned_types + float_types + bool_types:
@@ -119,7 +116,6 @@
             self.assertEqual(t(2L).value, 2.0)
             self.assertEqual(t(f).value, 2.0)
 
-    @xfail
     def test_integers(self):
         class FloatLike(object):
             def __float__(self):
diff --git a/lib-python/2.7/ctypes/test/test_objects.py b/lib-python/2.7/ctypes/test/test_objects.py
--- a/lib-python/2.7/ctypes/test/test_objects.py
+++ b/lib-python/2.7/ctypes/test/test_objects.py
@@ -22,7 +22,7 @@
 
 >>> array[4] = 'foo bar'
 >>> array._objects
-{'4': <CArgObject 'foo bar'>}
+{'4': 'foo bar'}
 >>> array[4]
 'foo bar'
 >>>
@@ -47,9 +47,9 @@
 
 >>> x.array[0] = 'spam spam spam'
 >>> x._objects
-{'0:2': <CArgObject 'spam spam spam'>}
+{'0:2': 'spam spam spam'}
 >>> x.array._b_base_._objects
-{'0:2': <CArgObject 'spam spam spam'>}
+{'0:2': 'spam spam spam'}
 >>>
 
 '''
diff --git a/lib-python/2.7/ctypes/test/test_parameters.py b/lib-python/2.7/ctypes/test/test_parameters.py
--- a/lib-python/2.7/ctypes/test/test_parameters.py
+++ b/lib-python/2.7/ctypes/test/test_parameters.py
@@ -1,7 +1,6 @@
 import unittest, sys
 from ctypes.test import need_symbol
-
-from ctypes.test import xfail
+import test.support
 
 class SimpleTypesTestCase(unittest.TestCase):
 
@@ -51,7 +50,6 @@
         self.assertEqual(CWCHARP.from_param("abc"), "abcabcabc")
 
     # XXX Replace by c_char_p tests
-    @xfail
     def test_cstrings(self):
         from ctypes import c_char_p, byref
 
@@ -85,10 +83,7 @@
 
         pa = c_wchar_p.from_param(c_wchar_p(u"123"))
         self.assertEqual(type(pa), c_wchar_p)
-    if sys.platform == "win32":
-        test_cw_strings = xfail(test_cw_strings)
 
-    @xfail
     def test_int_pointers(self):
         from ctypes import c_short, c_uint, c_int, c_long, POINTER, pointer
         LPINT = POINTER(c_int)
@@ -180,6 +175,36 @@
         # ArgumentError: argument 1: ValueError: 99
         self.assertRaises(ArgumentError, func, 99)
 
+    def test_abstract(self):
+        from ctypes import (Array, Structure, Union, _Pointer,
+                            _SimpleCData, _CFuncPtr)
+
+        self.assertRaises(TypeError, Array.from_param, 42)
+        self.assertRaises(TypeError, Structure.from_param, 42)
+        self.assertRaises(TypeError, Union.from_param, 42)
+        self.assertRaises(TypeError, _CFuncPtr.from_param, 42)
+        self.assertRaises(TypeError, _Pointer.from_param, 42)
+        self.assertRaises(TypeError, _SimpleCData.from_param, 42)
+
+    @test.support.cpython_only
+    def test_issue31311(self):
+        # __setstate__ should neither raise a SystemError nor crash in case
+        # of a bad __dict__.
+        from ctypes import Structure
+
+        class BadStruct(Structure):
+            @property

From pypy.commits at gmail.com  Sun May 19 16:15:07 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 19 May 2019 13:15:07 -0700 (PDT)
Subject: [pypy-commit] pypy vendor/stdlib-2.7.16: revert to older version
Message-ID: <5ce1b94b.1c69fb81.42ac1.4c9e@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: vendor/stdlib-2.7.16
Changeset: r96634:18a0680acbec
Date: 2019-05-19 22:48 +0300
http://bitbucket.org/pypy/pypy/changeset/18a0680acbec/

Log:	revert to older version

diff --git a/lib-python/2.7/ctypes/__init__.py b/lib-python/2.7/ctypes/__init__.py
--- a/lib-python/2.7/ctypes/__init__.py
+++ b/lib-python/2.7/ctypes/__init__.py
@@ -4,6 +4,7 @@
 
 __version__ = "1.1.0"
 
+import _ffi
 from _ctypes import Union, Structure, Array
 from _ctypes import _Pointer
 from _ctypes import CFuncPtr as _CFuncPtr
@@ -342,10 +343,6 @@
     """
     _func_flags_ = _FUNCFLAG_CDECL
     _func_restype_ = c_int
-    # default values for repr
-    _name = '<uninitialized>'
-    _handle = 0
-    _FuncPtr = None
 
     def __init__(self, name, mode=DEFAULT_MODE, handle=None,
                  use_errno=False,
@@ -363,9 +360,16 @@
         self._FuncPtr = _FuncPtr
 
         if handle is None:
-            self._handle = _dlopen(self._name, mode)
+            handle = 0
+        if flags & _FUNCFLAG_CDECL:
+            pypy_dll = _ffi.CDLL(name, mode, handle)
         else:
-            self._handle = handle
+            pypy_dll = _ffi.WinDLL(name, mode, handle)
+        self.__pypy_dll__ = pypy_dll
+        handle = int(pypy_dll)
+        if _sys.maxint > 2 ** 32:
+            handle = int(handle)   # long -> int
+        self._handle = handle
 
     def __repr__(self):
         return "<%s '%s', handle %x at %x>" % \
@@ -386,12 +390,13 @@
             func.__name__ = name_or_ordinal
         return func
 
-class PyDLL(CDLL):
-    """This class represents the Python library itself.  It allows
-    accessing Python API functions.  The GIL is not released, and
-    Python exceptions are handled correctly.
-    """
-    _func_flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI
+# Not in PyPy
+#class PyDLL(CDLL):
+#    """This class represents the Python library itself.  It allows
+#    accessing Python API functions.  The GIL is not released, and
+#    Python exceptions are handled correctly.
+#    """
+#    _func_flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI
 
 if _os.name in ("nt", "ce"):
 
@@ -444,15 +449,8 @@
         return self._dlltype(name)
 
 cdll = LibraryLoader(CDLL)
-pydll = LibraryLoader(PyDLL)
-
-if _os.name in ("nt", "ce"):
-    pythonapi = PyDLL("python dll", None, _sys.dllhandle)
-elif _sys.platform == "cygwin":
-    pythonapi = PyDLL("libpython%d.%d.dll" % _sys.version_info[:2])
-else:
-    pythonapi = PyDLL(None)
-
+# not on PyPy
+#pydll = LibraryLoader(PyDLL)
 
 if _os.name in ("nt", "ce"):
     windll = LibraryLoader(WinDLL)
@@ -498,9 +496,12 @@
         _flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI
     return CFunctionType
 
-_cast = PYFUNCTYPE(py_object, c_void_p, py_object, py_object)(_cast_addr)
 def cast(obj, typ):
-    return _cast(obj, obj, typ)
+    try:
+        c_void_p.from_param(obj)
+    except TypeError, e:
+        raise ArgumentError(str(e))
+    return _cast_addr(obj, obj, typ)
 
 _string_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr)
 def string_at(ptr, size=-1):

From pypy.commits at gmail.com  Sun May 19 16:15:09 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 19 May 2019 13:15:09 -0700 (PDT)
Subject: [pypy-commit] pypy vendor/stdlib-2.7.16: add _ssl.HAS_TLSv1_3
Message-ID: <5ce1b94d.1c69fb81.1a84a.d991@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: vendor/stdlib-2.7.16
Changeset: r96635:ea4bfedd2667
Date: 2019-05-19 23:14 +0300
http://bitbucket.org/pypy/pypy/changeset/ea4bfedd2667/

Log:	add _ssl.HAS_TLSv1_3

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -103,7 +103,10 @@
 constants["OPENSSL_VERSION_INFO"] = version_info
 constants["_OPENSSL_API_VERSION"] = version_info
 constants["OPENSSL_VERSION"] = SSLEAY_VERSION
-
+if OPENSSL_NO_TLS1_3:
+    constants["HAS_TLSv1_3"] = 0
+else:
+    constants["HAS_TLSv1_3"] = TLS1_3_VERSION
 
 def ssl_error(space, msg, errno=0, w_errtype=None, errcode=0):
     reason_str = None
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -42,6 +42,7 @@
         assert 'openssl' in lower_version or "libressl" in lower_version
 
         assert isinstance(_ssl.ALERT_DESCRIPTION_ACCESS_DENIED, int)
+        assert isinstance(_ssl.HAS_TLSv1_3, int)
 
     def test_RAND_add(self):
         import _ssl
diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -132,6 +132,8 @@
         SSL_OP_NO_TLSv1_1 = rffi_platform.ConstantInteger("SSL_OP_NO_TLSv1_1")
         SSL_OP_NO_TLSv1_2 = rffi_platform.ConstantInteger("SSL_OP_NO_TLSv1_2")
     OPENSSL_NO_TLSEXT = rffi_platform.Defined("OPENSSL_NO_TLSEXT")
+    OPENSSL_NO_TLS1_3 = rffi_platform.Defined("OPENSSL_NO_TLS1_3")
+    TLS1_3_VERSION = rffi_platform.Defined("TLS1_3_VERSION")
     SSL_OP_CIPHER_SERVER_PREFERENCE = rffi_platform.ConstantInteger(
         "SSL_OP_CIPHER_SERVER_PREFERENCE")
     SSL_OP_SINGLE_DH_USE = rffi_platform.ConstantInteger(

From pypy.commits at gmail.com  Sun May 19 16:30:40 2019
From: pypy.commits at gmail.com (andrewjlawrence)
Date: Sun, 19 May 2019 13:30:40 -0700 (PDT)
Subject: [pypy-commit] pypy winmultiprocessing: First implementation of
 socket.share. Fixed issue with _testcapi import.
Message-ID: <5ce1bcf0.1c69fb81.6bb06.0b73@mx.google.com>

Author: andrewjlawrence
Branch: winmultiprocessing
Changeset: r96636:440feb6ea372
Date: 2019-05-19 21:28 +0100
http://bitbucket.org/pypy/pypy/changeset/440feb6ea372/

Log:	First implementation of socket.share. Fixed issue with _testcapi
	import.

diff --git a/lib_pypy/_pypy_testcapi.py b/lib_pypy/_pypy_testcapi.py
--- a/lib_pypy/_pypy_testcapi.py
+++ b/lib_pypy/_pypy_testcapi.py
@@ -64,7 +64,6 @@
 
     compiler = new_compiler()
     compiler.output_dir = output_dir
-
     # Compile .c file
     include_dir = os.path.join(thisdir, '..', 'include')
     if sys.platform == 'win32':
@@ -80,7 +79,7 @@
     output_filename = modulename + _get_c_extension_suffix()
     if sys.platform == 'win32':
         libname = 'python{0[0]}{0[1]}'.format(sys.version_info)
-        library = os.path.join(thisdir, '..', 'lib', libname)
+        library = os.path.join(thisdir, '..', 'libs', libname)
         if not os.path.exists(library + '.lib'):
             # For a local translation or nightly build
             library = os.path.join(thisdir, '..', 'pypy', 'goal', libname)
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -16,6 +16,7 @@
     GetSetProperty, TypeDef, generic_new_descr, interp_attrproperty,
     make_weakref_descr)
 
+_WIN32 = sys.platform.startswith('win')
 
 # XXX Hack to separate rpython and pypy
 def addr_as_object(addr, fd, space):
@@ -197,13 +198,29 @@
             self.register_finalizer(space)
 
     @unwrap_spec(family=int, type=int, proto=int,
-                 w_fileno=WrappedDefault(None))
+                 w_fdobj=WrappedDefault(None))
     def descr_init(self, space, family=AF_INET, type=SOCK_STREAM, proto=0,
-                   w_fileno=None):
+                   w_fdobj=None):
         try:
-            if not space.is_w(w_fileno, space.w_None):
-                sock = RSocket(family, type, proto,
-                               fd=space.c_filedescriptor_w(w_fileno))
+            if not space.is_w(w_fdobj, space.w_None):
+                if _WIN32 and space.isinstance_w(w_fdobj, space.w_bytes):
+                    from rpython.rlib.rsocket import _c
+                    # it is possible to pass some bytes representing a socket
+                    # in the file descriptor object on winodws
+                    fdobj = space.text_w(w_fdobj)
+                    info_charptr = rffi.str2charp(fdobj)
+                    try:
+                        info_ptr = rffi.cast(lltype.Ptr(_c.WSAPROTOCOL_INFOW), info_charptr)
+                        fd = _c.WSASocketW(_c.FROM_PROTOCOL_INFO, _c.FROM_PROTOCOL_INFO,
+                        _c.FROM_PROTOCOL_INFO, info_ptr, 0, _c.WSA_FLAG_OVERLAPPED)
+                        if fd == rsocket.INVALID_SOCKET:
+                            raise converted_error(space, rsocket.last_error())
+                        sock = RSocket(family, type, proto, fd)
+                    finally:
+                        lltype.free(info_charptr, flavor='raw')
+                else:
+                    sock = RSocket(family, type, proto,
+                                   fd=space.c_filedescriptor_w(w_fdobj))
             else:
                 sock = RSocket(family, type, proto, inheritable=False)
             W_Socket.__init__(self, space, sock)
@@ -757,6 +774,26 @@
         finally:
             lltype.free(recv_ptr, flavor='raw')
 
+    @unwrap_spec(processid=int)
+    def share_w(self, space, processid):
+        from rpython.rtyper.lltypesystem import rffi, lltype
+        from rpython.rlib import rwin32
+        from rpython.rlib.rsocket import _c
+        info_ptr = lltype.malloc(_c.WSAPROTOCOL_INFOW, flavor='raw')
+        try:
+            winprocessid = rffi.cast(rwin32.DWORD, processid)
+            res = _c.WSADuplicateSocketW(
+                        self.sock.fd, winprocessid, info_ptr)
+
+            if res < 0:
+                raise converted_error(space, rsocket.last_error())
+
+            bytes_ptr = rffi.cast(rffi.CCHARP, info_ptr)
+            w_bytes = space.newbytes(rffi.charpsize2str(bytes_ptr, rffi.sizeof(_c.WSAPROTOCOL_INFOW)))
+        finally:
+            lltype.free(info_ptr, flavor='raw')
+        return w_bytes
+			
     @unwrap_spec(how="c_int")
     def shutdown_w(self, space, how):
         """shutdown(flag)
@@ -890,6 +927,7 @@
 """.split()
 if hasattr(rsocket._c, 'WSAIoctl'):
     socketmethodnames.append('ioctl')
+    socketmethodnames.append('share')
 if rsocket._c.HAVE_SENDMSG:
     socketmethodnames.append('sendmsg')
     socketmethodnames.append('recvmsg')
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -574,6 +574,13 @@
         s = _socket.socket()
         raises(ValueError, s.ioctl, -1, None)
         s.ioctl(_socket.SIO_KEEPALIVE_VALS, (1, 100, 100))
+		
+    def test_socket_sharelocal(self):
+        import _socket, sys, os
+        if sys.platform != 'win32':
+            skip("win32 only")
+        assert hasattr(_socket.socket, 'share')
+        s = _socket.socket()
 
     def test_dup(self):
         import _socket as socket, os
diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py
--- a/rpython/rlib/_rsocket_rffi.py
+++ b/rpython/rlib/_rsocket_rffi.py
@@ -203,6 +203,7 @@
 FD_CONNECT_BIT FD_CLOSE_BIT
 WSA_IO_PENDING WSA_IO_INCOMPLETE WSA_INVALID_HANDLE
 WSA_INVALID_PARAMETER WSA_NOT_ENOUGH_MEMORY WSA_OPERATION_ABORTED
+WSA_FLAG_OVERLAPPED
 SIO_RCVALL SIO_KEEPALIVE_VALS
 
 SIOCGIFNAME SIOCGIFINDEX
@@ -1009,6 +1010,7 @@
     CConfig.WSAPROTOCOL_INFO = platform.Struct(
         'WSAPROTOCOL_INFOA',
         [])  # Struct is just passed between functions
+	   
     CConfig.FROM_PROTOCOL_INFO = platform.DefinedConstantInteger(
         'FROM_PROTOCOL_INFO')
 
@@ -1033,6 +1035,45 @@
         [('onoff', rffi.ULONG),
          ('keepalivetime', rffi.ULONG),
          ('keepaliveinterval', rffi.ULONG)])
+	
+    CConfig.GUID = platform.Struct(
+		     'struct _GUID',
+			 [('Data1', rffi.UINT),
+             ('Data2', rffi.UINT),
+             ('Data3', rffi.UINT),
+             ('Data4', rffi.CFixedArray(rffi.UCHAR, 8))
+         ])
+		 
+    CConfig.WSAPROTOCOLCHAIN = platform.Struct(
+        'struct _WSAPROTOCOLCHAIN',
+        [('ChainLen', rffi.INT),
+         ('ChainEntries', rffi.CFixedArray(rffi.UINT, 7))])
+	
+    WSAPROTOCOLCHAIN = CConfig.WSAPROTOCOLCHAIN
+    GUID = CConfig.GUID 
+	
+    CConfig.WSAPROTOCOL_INFOW = platform.Struct(
+        'struct _WSAPROTOCOL_INFOW',
+        [('dwServiceFlags1', rffi.UINT),
+         ('dwServiceFlags2', rffi.UINT),
+         ('dwServiceFlags3', rffi.UINT),
+         ('dwServiceFlags4', rffi.UINT),
+         ('dwProviderFlags', rffi.UINT),
+         ('ProviderId', GUID),
+         ('dwCatalogEntryId', rffi.UINT),
+         ('ProtocolChain', WSAPROTOCOLCHAIN),
+         ('iVersion', rffi.INT),
+         ('iAddressFamily', rffi.INT),
+         ('iMaxSockAddr', rffi.INT),
+         ('iMinSockAddr', rffi.INT),
+         ('iSocketType', rffi.INT),
+         ('iProtocol', rffi.INT),
+         ('iProtocolMaxOffset', rffi.INT),
+         ('iNetworkByteOrder', rffi.INT),
+         ('iSecurityScheme', rffi.INT),
+         ('dwMessageSize', rffi.UINT),
+         ('dwProviderReserved', rffi.UINT),
+         ('szProtocol',  rffi.CFixedArray(rffi.UCHAR, 256))])
 
 
 class cConfig:
@@ -1336,6 +1377,20 @@
                          rffi.VOIDP, rwin32.DWORD,
                          rwin32.LPDWORD, rffi.VOIDP, rffi.VOIDP],
                         rffi.INT, save_err=SAVE_ERR)
+
+    WSAPROTOCOL_INFOW = cConfig.WSAPROTOCOL_INFOW
+
+    WSADuplicateSocketW = external('WSADuplicateSocketW',
+                                 [socketfd_type, rwin32.DWORD,
+                                  lltype.Ptr(WSAPROTOCOL_INFOW)],
+                                  rffi.INT, save_err=SAVE_ERR)
+
+    WSASocketW = external('WSASocketW',
+                         [rffi.INT, rffi.INT, rffi.INT,
+                          lltype.Ptr(WSAPROTOCOL_INFOW),
+                          rwin32.DWORD, rwin32.DWORD],
+                         socketfd_type, save_err=SAVE_ERR)
+								  
     tcp_keepalive = cConfig.tcp_keepalive
 
     WSAPROTOCOL_INFO = cConfig.WSAPROTOCOL_INFO

From pypy.commits at gmail.com  Mon May 20 01:04:14 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 19 May 2019 22:04:14 -0700 (PDT)
Subject: [pypy-commit] pypy vendor/stdlib-2.7.16: implement
 weakref._remove_dead_weakref, change pip version in ensurepip
Message-ID: <5ce2354e.1c69fb81.1dd7d.5bf7@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: vendor/stdlib-2.7.16
Changeset: r96637:819d32f3f951
Date: 2019-05-20 07:51 +0300
http://bitbucket.org/pypy/pypy/changeset/819d32f3f951/

Log:	implement weakref._remove_dead_weakref, change pip version in
	ensurepip

diff --git a/lib-python/2.7/ensurepip/__init__.py b/lib-python/2.7/ensurepip/__init__.py
--- a/lib-python/2.7/ensurepip/__init__.py
+++ b/lib-python/2.7/ensurepip/__init__.py
@@ -14,7 +14,7 @@
 
 _SETUPTOOLS_VERSION = "40.6.2"
 
-_PIP_VERSION = "18.1"
+_PIP_VERSION = "19.1.1"
 
 _PROJECTS = [
     ("setuptools", _SETUPTOOLS_VERSION),
diff --git a/pypy/module/_weakref/interp__weakref.py b/pypy/module/_weakref/interp__weakref.py
--- a/pypy/module/_weakref/interp__weakref.py
+++ b/pypy/module/_weakref/interp__weakref.py
@@ -313,8 +313,19 @@
     if not space.isinstance_w(w_dict, space.w_dict):
         raise oefmt(space.w_TypeError,
             '_remove_dead_weakref() argument 1 must be dict, not %R', w_dict)
-    raise oefmt(space.w_NotImplementedError, "Not implemented yet") 
-    # CPython uses a PyDict_DelItemIf(dct, key, is_dead_weakref)
+    # The call is supposed to be "atomic"
+    try:
+        w_value = space.getitem(w_dict, w_key)
+    except KeyError:
+        return 
+    if w_value is None:
+        return
+    if not space.isinstance_w(w_value, W_WeakrefBase):
+        raise oefmt(space.w_TypeError, "not a weakref")
+    w_ref = w_value.dereference()
+    if w_ref is None or space.isinstance_w(w_ref, space.w_None):
+        return
+    space.delitem(w_dict, w_key) 
 
 #_________________________________________________________________
 # Proxy

From pypy.commits at gmail.com  Mon May 20 01:04:16 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 19 May 2019 22:04:16 -0700 (PDT)
Subject: [pypy-commit] pypy vendor/stdlib-2.7.16: revert sysconfig,
 update info and how-to-update
Message-ID: <5ce23550.1c69fb81.bf2f8.4875@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: vendor/stdlib-2.7.16
Changeset: r96638:3a0d737b20b0
Date: 2019-05-20 08:03 +0300
http://bitbucket.org/pypy/pypy/changeset/3a0d737b20b0/

Log:	revert sysconfig, update info and how-to-update

diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py
--- a/lib-python/2.7/sysconfig.py
+++ b/lib-python/2.7/sysconfig.py
@@ -7,22 +7,32 @@
 
 _INSTALL_SCHEMES = {
     'posix_prefix': {
-        'stdlib': '{base}/lib/python{py_version_short}',
-        'platstdlib': '{platbase}/lib/python{py_version_short}',
-        'purelib': '{base}/lib/python{py_version_short}/site-packages',
-        'platlib': '{platbase}/lib/python{py_version_short}/site-packages',
-        'include': '{base}/include/python{py_version_short}',
-        'platinclude': '{platbase}/include/python{py_version_short}',
+        'stdlib': '{base}/lib/{implementation_lower}{py_version_short}',
+        'platstdlib': '{platbase}/lib/{implementation_lower}{py_version_short}',
+        'purelib': '{base}/lib/{implementation_lower}{py_version_short}/site-packages',
+        'platlib': '{platbase}/lib/{implementation_lower}{py_version_short}/site-packages',
+        'include': '{base}/include/{implementation_lower}{py_version_short}',
+        'platinclude': '{platbase}/include/{implementation_lower}{py_version_short}',
         'scripts': '{base}/bin',
         'data': '{base}',
         },
     'posix_home': {
-        'stdlib': '{base}/lib/python',
-        'platstdlib': '{base}/lib/python',
-        'purelib': '{base}/lib/python',
-        'platlib': '{base}/lib/python',
-        'include': '{base}/include/python',
-        'platinclude': '{base}/include/python',
+        'stdlib': '{base}/lib/{implementation_lower}',
+        'platstdlib': '{base}/lib/{implementation_lower}',
+        'purelib': '{base}/lib/{implementation_lower}',
+        'platlib': '{base}/lib/{implementation_lower}',
+        'include': '{base}/include/{implementation_lower}',
+        'platinclude': '{base}/include/{implementation_lower}',
+        'scripts': '{base}/bin',
+        'data'   : '{base}',
+        },
+    'pypy': {
+        'stdlib': '{base}/lib-{implementation_lower}/{py_version_short}',
+        'platstdlib': '{base}/lib-{implementation_lower}/{py_version_short}',
+        'purelib': '{base}/site-packages',
+        'platlib': '{base}/site-packages',
+        'include': '{base}/include',
+        'platinclude': '{base}/include',
         'scripts': '{base}/bin',
         'data'   : '{base}',
         },
@@ -47,37 +57,37 @@
         'data'   : '{base}',
         },
     'os2_home': {
-        'stdlib': '{userbase}/lib/python{py_version_short}',
-        'platstdlib': '{userbase}/lib/python{py_version_short}',
-        'purelib': '{userbase}/lib/python{py_version_short}/site-packages',
-        'platlib': '{userbase}/lib/python{py_version_short}/site-packages',
-        'include': '{userbase}/include/python{py_version_short}',
+        'stdlib': '{userbase}/lib/{implementation_lower}{py_version_short}',
+        'platstdlib': '{userbase}/lib/{implementation_lower}{py_version_short}',
+        'purelib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages',
+        'platlib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages',
+        'include': '{userbase}/include/{implementation_lower}{py_version_short}',
         'scripts': '{userbase}/bin',
         'data'   : '{userbase}',
         },
     'nt_user': {
-        'stdlib': '{userbase}/Python{py_version_nodot}',
-        'platstdlib': '{userbase}/Python{py_version_nodot}',
-        'purelib': '{userbase}/Python{py_version_nodot}/site-packages',
-        'platlib': '{userbase}/Python{py_version_nodot}/site-packages',
-        'include': '{userbase}/Python{py_version_nodot}/Include',
+        'stdlib': '{userbase}/{implementation}{py_version_nodot}',
+        'platstdlib': '{userbase}/{implementation}{py_version_nodot}',
+        'purelib': '{userbase}/{implementation}{py_version_nodot}/site-packages',
+        'platlib': '{userbase}/{implementation}{py_version_nodot}/site-packages',
+        'include': '{userbase}/{implementation}{py_version_nodot}/Include',
         'scripts': '{userbase}/Scripts',
         'data'   : '{userbase}',
         },
     'posix_user': {
-        'stdlib': '{userbase}/lib/python{py_version_short}',
-        'platstdlib': '{userbase}/lib/python{py_version_short}',
-        'purelib': '{userbase}/lib/python{py_version_short}/site-packages',
-        'platlib': '{userbase}/lib/python{py_version_short}/site-packages',
-        'include': '{userbase}/include/python{py_version_short}',
+        'stdlib': '{userbase}/lib/{implementation_lower}{py_version_short}',
+        'platstdlib': '{userbase}/lib/{implementation_lower}{py_version_short}',
+        'purelib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages',
+        'platlib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages',
+        'include': '{userbase}/include/{implementation_lower}{py_version_short}',
         'scripts': '{userbase}/bin',
         'data'   : '{userbase}',
         },
     'osx_framework_user': {
-        'stdlib': '{userbase}/lib/python',
-        'platstdlib': '{userbase}/lib/python',
-        'purelib': '{userbase}/lib/python/site-packages',
-        'platlib': '{userbase}/lib/python/site-packages',
+        'stdlib': '{userbase}/lib/{implementation_lower}',
+        'platstdlib': '{userbase}/lib/{implementation_lower}',
+        'purelib': '{userbase}/lib/{implementation_lower}/site-packages',
+        'platlib': '{userbase}/lib/{implementation_lower}/site-packages',
         'include': '{userbase}/include',
         'scripts': '{userbase}/bin',
         'data'   : '{userbase}',
@@ -94,6 +104,11 @@
 _CONFIG_VARS = None
 _USER_BASE = None
 
+def _get_implementation():
+    if '__pypy__' in sys.builtin_module_names:
+        return 'PyPy'
+    return 'Python'
+
 def _safe_realpath(path):
     try:
         return realpath(path)
@@ -112,11 +127,6 @@
 # PC/VS7.1
 if os.name == "nt" and "\\pc\\v" in _PROJECT_BASE[-10:].lower():
     _PROJECT_BASE = _safe_realpath(os.path.join(_PROJECT_BASE, pardir, pardir))
-# PC/VS9.0/amd64
-if (os.name == "nt"
-   and os.path.basename(os.path.dirname(os.path.dirname(_PROJECT_BASE))).lower() == "pc"
-   and os.path.basename(os.path.dirname(_PROJECT_BASE)).lower() == "vs9.0"):
-    _PROJECT_BASE = _safe_realpath(os.path.join(_PROJECT_BASE, pardir, pardir, pardir))
 # PC/AMD64
 if os.name == "nt" and "\\pcbuild\\amd64" in _PROJECT_BASE[-14:].lower():
     _PROJECT_BASE = _safe_realpath(os.path.join(_PROJECT_BASE, pardir, pardir))
@@ -167,7 +177,9 @@
     return res
 
 def _get_default_scheme():
-    if os.name == 'posix':
+    if '__pypy__' in sys.builtin_module_names:
+        return 'pypy'
+    elif os.name == 'posix':
         # the default scheme for posix is posix_prefix
         return 'posix_prefix'
     return os.name
@@ -357,7 +369,6 @@
 
 def _init_posix(vars):
     """Initialize the module as appropriate for POSIX systems."""
-    # _sysconfigdata is generated at build time, see _generate_posix_vars()
     from _sysconfigdata import build_time_vars
     vars.update(build_time_vars)
 
@@ -470,6 +481,9 @@
         _CONFIG_VARS['base'] = _PREFIX
         _CONFIG_VARS['platbase'] = _EXEC_PREFIX
         _CONFIG_VARS['projectbase'] = _PROJECT_BASE
+        _CONFIG_VARS['implementation'] = _get_implementation()
+        _CONFIG_VARS['implementation_lower'] = _get_implementation().lower()
+        _CONFIG_VARS['LIBRARY'] = ''
 
         if os.name in ('nt', 'os2'):
             _init_non_posix(_CONFIG_VARS)
@@ -508,6 +522,15 @@
             import _osx_support
             _osx_support.customize_config_vars(_CONFIG_VARS)
 
+        # PyPy:
+        import imp
+        for suffix, mode, type_ in imp.get_suffixes():
+            if type_ == imp.C_EXTENSION:
+                _CONFIG_VARS['SOABI'] = suffix.split('.')[1]
+                break
+        _CONFIG_VARS['INCLUDEPY'] = os.path.join(_CONFIG_VARS['prefix'],
+                                                 'include')
+
     if args:
         vals = []
         for name in args:
@@ -639,6 +662,8 @@
     _print_dict('Paths', get_paths())
     print
     _print_dict('Variables', get_config_vars())
+    print
+    _print_dict('User', get_paths('%s_user' % os.name))
 
 
 if __name__ == '__main__':
diff --git a/lib-python/stdlib-upgrade.txt b/lib-python/stdlib-upgrade.txt
--- a/lib-python/stdlib-upgrade.txt
+++ b/lib-python/stdlib-upgrade.txt
@@ -15,6 +15,7 @@
    2d. hg remove --after
    2e. show copied files in cpython repo by running `hg diff --git -r v<old> -r v<new> Lib | grep '^copy \(from\|to\)'`
    2f. fix copies / renames manually by running `hg copy --after <from> <to>` for each copied file
+   2g. fix up (revert?) sysconfig.py, distutils/sysconfig*.py
 3. update stdlib-version.txt with the output of hg -id from the cpython repo
 4. commit
 5. update to default / py3k
diff --git a/lib-python/stdlib-version.txt b/lib-python/stdlib-version.txt
--- a/lib-python/stdlib-version.txt
+++ b/lib-python/stdlib-version.txt
@@ -1,9 +1,9 @@
 in here there are copies of the Lib/ directory of the CPython HG repository
 at http://hg.python.org/cpython/
 
-the outputs for hg id of each are:
+the outputs for "hg id" or "git rev-parse HEAD; git status" of each are:
 
     2.7::
-        a06454b1afa1 (2.7) v2.7.13
+        413a49145e35c7a2d3a7de27c5a1828449c9b2e5 v2.7.16
     3::
         cef745775b65 (3.2) v3.2.5

From pypy.commits at gmail.com  Mon May 20 02:13:23 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 19 May 2019 23:13:23 -0700 (PDT)
Subject: [pypy-commit] pypy vendor/stdlib-2.7.16: revert pickle.py to pypy
 version, fix _remove_dead_weakref
Message-ID: <5ce24583.1c69fb81.22dca.1d56@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: vendor/stdlib-2.7.16
Changeset: r96639:9aa8f255fa02
Date: 2019-05-20 09:12 +0300
http://bitbucket.org/pypy/pypy/changeset/9aa8f255fa02/

Log:	revert pickle.py to pypy version, fix _remove_dead_weakref

diff --git a/lib-python/2.7/pickle.py b/lib-python/2.7/pickle.py
--- a/lib-python/2.7/pickle.py
+++ b/lib-python/2.7/pickle.py
@@ -168,7 +168,7 @@
 
 # Pickling machinery
 
-class Pickler:
+class Pickler(object):
 
     def __init__(self, file, protocol=None):
         """This takes a file-like object for writing a pickle data stream.
@@ -351,7 +351,9 @@
             raise PicklingError("args from reduce() should be a tuple")
 
         # Assert that func is callable
-        if not hasattr(func, '__call__'):
+        try:
+            func.__call__
+        except AttributeError:
             raise PicklingError("func from reduce should be callable")
 
         save = self.save
@@ -644,6 +646,10 @@
             # else tmp is empty, and we're done
 
     def save_dict(self, obj):
+        modict_saver = self._pickle_maybe_moduledict(obj)
+        if modict_saver is not None:
+            return self.save_reduce(*modict_saver)
+
         write = self.write
 
         if self.bin:
@@ -693,6 +699,23 @@
                 write(SETITEM)
             # else tmp is empty, and we're done
 
+    def _pickle_maybe_moduledict(self, obj):
+        # save module dictionary as "getattr(module, '__dict__')"
+        try:
+            name = obj['__name__']
+            if type(name) is not str:
+                return None
+            themodule = sys.modules[name]
+            if type(themodule) is not ModuleType:
+                return None
+            if themodule.__dict__ is not obj:
+                return None
+        except (AttributeError, KeyError, TypeError):
+            return None
+
+        return getattr, (themodule, '__dict__')
+
+
     def save_inst(self, obj):
         cls = obj.__class__
 
@@ -733,6 +756,29 @@
 
     dispatch[InstanceType] = save_inst
 
+    def save_function(self, obj):
+        try:
+            return self.save_global(obj)
+        except PicklingError, e:
+            pass
+        # Check copy_reg.dispatch_table
+        reduce = dispatch_table.get(type(obj))
+        if reduce:
+            rv = reduce(obj)
+        else:
+            # Check for a __reduce_ex__ method, fall back to __reduce__
+            reduce = getattr(obj, "__reduce_ex__", None)
+            if reduce:
+                rv = reduce(self.proto)
+            else:
+                reduce = getattr(obj, "__reduce__", None)
+                if reduce:
+                    rv = reduce()
+                else:
+                    raise e
+        return self.save_reduce(obj=obj, *rv)
+    dispatch[FunctionType] = save_function
+
     def save_global(self, obj, name=None, pack=struct.pack):
         write = self.write
         memo = self.memo
@@ -774,7 +820,6 @@
         self.memoize(obj)
 
     dispatch[ClassType] = save_global
-    dispatch[FunctionType] = save_global
     dispatch[BuiltinFunctionType] = save_global
     dispatch[TypeType] = save_global
 
@@ -830,7 +875,7 @@
 
 # Unpickling machinery
 
-class Unpickler:
+class Unpickler(object):
 
     def __init__(self, file):
         """This takes a file-like object for reading a pickle data stream.
@@ -1339,6 +1384,7 @@
 
 def decode_long(data):
     r"""Decode a long from a two's complement little-endian binary string.
+    This is overriden on PyPy by a RPython version that has linear complexity.
 
     >>> decode_long('')
     0L
@@ -1365,6 +1411,11 @@
         n -= 1L << (nbytes * 8)
     return n
 
+try:
+    from __pypy__ import decode_long
+except ImportError:
+    pass
+
 # Shorthands
 
 try:
diff --git a/pypy/module/_weakref/interp__weakref.py b/pypy/module/_weakref/interp__weakref.py
--- a/pypy/module/_weakref/interp__weakref.py
+++ b/pypy/module/_weakref/interp__weakref.py
@@ -320,7 +320,7 @@
         return 
     if w_value is None:
         return
-    if not space.isinstance_w(w_value, W_WeakrefBase):
+    if not isinstance(w_value, W_WeakrefBase):
         raise oefmt(space.w_TypeError, "not a weakref")
     w_ref = w_value.dereference()
     if w_ref is None or space.isinstance_w(w_ref, space.w_None):

From pypy.commits at gmail.com  Mon May 20 07:30:35 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 20 May 2019 04:30:35 -0700 (PDT)
Subject: [pypy-commit] pypy vendor/stdlib-2.7.16: abandon branch
Message-ID: <5ce28fdb.1c69fb81.42ac1.d50b@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: vendor/stdlib-2.7.16
Changeset: r96640:9c5ebfaa4643
Date: 2019-05-20 11:27 +0300
http://bitbucket.org/pypy/pypy/changeset/9c5ebfaa4643/

Log:	abandon branch

diff --git a/lib-python/2.7/UserDict.py b/lib-python/2.7/UserDict.py
--- a/lib-python/2.7/UserDict.py
+++ b/lib-python/2.7/UserDict.py
@@ -113,8 +113,12 @@
     def __iter__(self):
         return iter(self.data)
 
-import _abcoll
-_abcoll.MutableMapping.register(IterableUserDict)
+try:
+    import _abcoll
+except ImportError:
+    pass    # e.g. no '_weakref' module on this pypy
+else:
+    _abcoll.MutableMapping.register(IterableUserDict)
 
 
 class DictMixin:
diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py
--- a/lib-python/2.7/subprocess.py
+++ b/lib-python/2.7/subprocess.py
@@ -71,10 +71,6 @@
 else:
     import select
     _has_poll = hasattr(select, 'poll')
-    try:
-        import threading
-    except ImportError:
-        threading = None
     import fcntl
     import pickle
 
@@ -339,6 +335,21 @@
         """Create new Popen instance."""
         _cleanup()
 
+        # --- PyPy hack, see _pypy_install_libs_after_virtualenv() ---
+        # match arguments passed by different versions of virtualenv
+        if args[1:] in (
+            ['-c', 'import sys; print(sys.prefix)'],        # 1.6 10ba3f3c
+            ['-c', "\nimport sys\nprefix = sys.prefix\n"    # 1.7 0e9342ce
+             "if sys.version_info[0] == 3:\n"
+             "    prefix = prefix.encode('utf8')\n"
+             "if hasattr(sys.stdout, 'detach'):\n"
+             "    sys.stdout = sys.stdout.detach()\n"
+             "elif hasattr(sys.stdout, 'buffer'):\n"
+             "    sys.stdout = sys.stdout.buffer\nsys.stdout.write(prefix)\n"],
+            ['-c', 'import sys;out=sys.stdout;getattr(out, "buffer"'
+             ', out).write(sys.prefix.encode("utf-8"))']):  # 1.7.2 a9454bce
+            _pypy_install_libs_after_virtualenv(args[0])
+
         if not isinstance(bufsize, (int, long)):
             raise TypeError("bufsize must be an integer")
 
@@ -505,54 +516,63 @@
             c2pread, c2pwrite = None, None
             errread, errwrite = None, None
 
+            ispread = False
             if stdin is None:
                 p2cread = _subprocess.GetStdHandle(_subprocess.STD_INPUT_HANDLE)
                 if p2cread is None:
                     p2cread, _ = _subprocess.CreatePipe(None, 0)
+                    ispread = True
             elif stdin == PIPE:
                 p2cread, p2cwrite = _subprocess.CreatePipe(None, 0)
-            elif isinstance(stdin, (int, long)):
+                ispread = True
+            elif isinstance(stdin, int):
                 p2cread = msvcrt.get_osfhandle(stdin)
             else:
                 # Assuming file-like object
                 p2cread = msvcrt.get_osfhandle(stdin.fileno())
-            p2cread = self._make_inheritable(p2cread)
+            p2cread = self._make_inheritable(p2cread, ispread)
             # We just duplicated the handle, it has to be closed at the end
             to_close.add(p2cread)
             if stdin == PIPE:
                 to_close.add(p2cwrite)
 
+            ispwrite = False
             if stdout is None:
                 c2pwrite = _subprocess.GetStdHandle(_subprocess.STD_OUTPUT_HANDLE)
                 if c2pwrite is None:
                     _, c2pwrite = _subprocess.CreatePipe(None, 0)
+                    ispwrite = True
             elif stdout == PIPE:
                 c2pread, c2pwrite = _subprocess.CreatePipe(None, 0)
-            elif isinstance(stdout, (int, long)):
+                ispwrite = True
+            elif isinstance(stdout, int):
                 c2pwrite = msvcrt.get_osfhandle(stdout)
             else:
                 # Assuming file-like object
                 c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
-            c2pwrite = self._make_inheritable(c2pwrite)
+            c2pwrite = self._make_inheritable(c2pwrite, ispwrite)
             # We just duplicated the handle, it has to be closed at the end
             to_close.add(c2pwrite)
             if stdout == PIPE:
                 to_close.add(c2pread)
 
+            ispwrite = False
             if stderr is None:
                 errwrite = _subprocess.GetStdHandle(_subprocess.STD_ERROR_HANDLE)
                 if errwrite is None:
                     _, errwrite = _subprocess.CreatePipe(None, 0)
+                    ispwrite = True
             elif stderr == PIPE:
                 errread, errwrite = _subprocess.CreatePipe(None, 0)
+                ispwrite = True
             elif stderr == STDOUT:
                 errwrite = c2pwrite
-            elif isinstance(stderr, (int, long)):
+            elif isinstance(stderr, int):
                 errwrite = msvcrt.get_osfhandle(stderr)
             else:
                 # Assuming file-like object
                 errwrite = msvcrt.get_osfhandle(stderr.fileno())
-            errwrite = self._make_inheritable(errwrite)
+            errwrite = self._make_inheritable(errwrite, ispwrite)
             # We just duplicated the handle, it has to be closed at the end
             to_close.add(errwrite)
             if stderr == PIPE:
@@ -563,11 +583,16 @@
                     errread, errwrite), to_close
 
 
-        def _make_inheritable(self, handle):
+        def _make_inheritable(self, handle, close=False):
             """Return a duplicate of handle, which is inheritable"""
-            return _subprocess.DuplicateHandle(_subprocess.GetCurrentProcess(),
+            dupl = _subprocess.DuplicateHandle(_subprocess.GetCurrentProcess(),
                                 handle, _subprocess.GetCurrentProcess(), 0, 1,
                                 _subprocess.DUPLICATE_SAME_ACCESS)
+            # PyPy: If the initial handle was obtained with CreatePipe,
+            # close it.
+            if close:
+                handle.Close()
+            return dupl
 
 
         def _find_w9xpopen(self):
@@ -724,11 +749,10 @@
                         if e.errno == errno.EPIPE:
                             # communicate() should ignore broken pipe error
                             pass
-                        elif e.errno == errno.EINVAL:
-                            # bpo-19612, bpo-30418: On Windows, stdin.write()
-                            # fails with EINVAL if the child process exited or
-                            # if the child process is still running but closed
-                            # the pipe.
+                        elif (e.errno == errno.EINVAL
+                              and self.poll() is not None):
+                            # Issue #19612: stdin.write() fails with EINVAL
+                            # if the process already exited before the write
                             pass
                         else:
                             raise
@@ -805,7 +829,7 @@
             elif stdin == PIPE:
                 p2cread, p2cwrite = self.pipe_cloexec()
                 to_close.update((p2cread, p2cwrite))
-            elif isinstance(stdin, (int, long)):
+            elif isinstance(stdin, int):
                 p2cread = stdin
             else:
                 # Assuming file-like object
@@ -816,7 +840,7 @@
             elif stdout == PIPE:
                 c2pread, c2pwrite = self.pipe_cloexec()
                 to_close.update((c2pread, c2pwrite))
-            elif isinstance(stdout, (int, long)):
+            elif isinstance(stdout, int):
                 c2pwrite = stdout
             else:
                 # Assuming file-like object
@@ -832,7 +856,7 @@
                     errwrite = c2pwrite
                 else: # child's stdout is not set, use parent's stdout
                     errwrite = sys.__stdout__.fileno()
-            elif isinstance(stderr, (int, long)):
+            elif isinstance(stderr, int):
                 errwrite = stderr
             else:
                 # Assuming file-like object
@@ -882,21 +906,6 @@
                         pass
 
 
-        # Used as a bandaid workaround for https://bugs.python.org/issue27448
-        # to prevent multiple simultaneous subprocess launches from interfering
-        # with one another and leaving gc disabled.
-        if threading:
-            _disabling_gc_lock = threading.Lock()
-        else:
-            class _noop_context_manager(object):
-                # A dummy context manager that does nothing for the rare
-                # user of a --without-threads build.
-                def __enter__(self): pass
-                def __exit__(self, *args): pass
-
-            _disabling_gc_lock = _noop_context_manager()
-
-
         def _execute_child(self, args, executable, preexec_fn, close_fds,
                            cwd, env, universal_newlines,
                            startupinfo, creationflags, shell, to_close,
@@ -928,12 +937,10 @@
             errpipe_read, errpipe_write = self.pipe_cloexec()
             try:
                 try:
-                    with self._disabling_gc_lock:
-                        gc_was_enabled = gc.isenabled()
-                        # Disable gc to avoid bug where gc -> file_dealloc ->
-                        # write to stderr -> hang.
-                        # https://bugs.python.org/issue1336
-                        gc.disable()
+                    gc_was_enabled = gc.isenabled()
+                    # Disable gc to avoid bug where gc -> file_dealloc ->
+                    # write to stderr -> hang.  http://bugs.python.org/issue1336
+                    gc.disable()
                     try:
                         self.pid = os.fork()
                     except:
@@ -1007,10 +1014,9 @@
                             exc_value.child_traceback = ''.join(exc_lines)
                             os.write(errpipe_write, pickle.dumps(exc_value))
 
-                        finally:
-                            # This exitcode won't be reported to applications, so it
-                            # really doesn't matter what we return.
-                            os._exit(255)
+                        # This exitcode won't be reported to applications, so it
+                        # really doesn't matter what we return.
+                        os._exit(255)
 
                     # Parent
                     if gc_was_enabled:
@@ -1049,16 +1055,13 @@
 
         def _handle_exitstatus(self, sts, _WIFSIGNALED=os.WIFSIGNALED,
                 _WTERMSIG=os.WTERMSIG, _WIFEXITED=os.WIFEXITED,
-                _WEXITSTATUS=os.WEXITSTATUS, _WIFSTOPPED=os.WIFSTOPPED,
-                _WSTOPSIG=os.WSTOPSIG):
+                _WEXITSTATUS=os.WEXITSTATUS):
             # This method is called (indirectly) by __del__, so it cannot
             # refer to anything outside of its local scope.
             if _WIFSIGNALED(sts):
                 self.returncode = -_WTERMSIG(sts)
             elif _WIFEXITED(sts):
                 self.returncode = _WEXITSTATUS(sts)
-            elif _WIFSTOPPED(sts):
-                self.returncode = -_WSTOPSIG(sts)
             else:
                 # Should never happen
                 raise RuntimeError("Unknown child exit status!")
@@ -1279,6 +1282,27 @@
             self.send_signal(signal.SIGKILL)
 
 
+def _pypy_install_libs_after_virtualenv(target_executable):
+    # https://bitbucket.org/pypy/pypy/issue/1922/future-proofing-virtualenv
+    #
+    # PyPy 2.4.1 turned --shared on by default.  This means the pypy binary
+    # depends on the 'libpypy-c.so' shared library to be able to run.
+    # The virtualenv code existing at the time did not account for this
+    # and would break.  Try to detect that we're running under such a
+    # virtualenv in the "Testing executable with" phase and copy the
+    # library ourselves.
+    caller = sys._getframe(2)
+    if ('virtualenv_version' in caller.f_globals and
+                  'copyfile' in caller.f_globals):
+        dest_dir = sys.pypy_resolvedirof(target_executable)
+        src_dir = sys.pypy_resolvedirof(sys.executable)
+        for libname in ['libpypy-c.so', 'libpypy-c.dylib', 'libpypy-c.dll']:
+            dest_library = os.path.join(dest_dir, libname)
+            src_library = os.path.join(src_dir, libname)
+            if os.path.exists(src_library):
+                caller.f_globals['copyfile'](src_library, dest_library)
+
+
 def _demo_posix():
     #
     # Example 1: Simple redirection: Get process list
diff --git a/lib-python/stdlib-upgrade.txt b/lib-python/stdlib-upgrade.txt
--- a/lib-python/stdlib-upgrade.txt
+++ b/lib-python/stdlib-upgrade.txt
@@ -5,6 +5,16 @@
 
     overly detailed
 
+The idea is to 
+- exactly copy cpython's stdlib to a "vendor" branch on top of the previous
+  version
+- clean up the hg history for moved files so the merge from pypy will work
+- branch off that with a new integration branch
+- merge default or py3 into that branch, which will update with all the
+  modifications pypy made to the stdlib
+
+And in more detail:
+
 0. make sure your working dir is clean
 1. check out the branch vendor/stdlib (for 2.7) or vendor/stdlib-3-* (for py3k)
    or create branch vendor/stdlib-3-*
@@ -15,7 +25,6 @@
    2d. hg remove --after
    2e. show copied files in cpython repo by running `hg diff --git -r v<old> -r v<new> Lib | grep '^copy \(from\|to\)'`
    2f. fix copies / renames manually by running `hg copy --after <from> <to>` for each copied file
-   2g. fix up (revert?) sysconfig.py, distutils/sysconfig*.py
 3. update stdlib-version.txt with the output of hg -id from the cpython repo
 4. commit
 5. update to default / py3k

From pypy.commits at gmail.com  Mon May 20 07:30:40 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 20 May 2019 04:30:40 -0700 (PDT)
Subject: [pypy-commit] pypy vendor/stdlib: add how-to-update
Message-ID: <5ce28fe0.1c69fb81.50fe6.2c4a@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: vendor/stdlib
Changeset: r96642:873173b17a5e
Date: 2019-05-20 12:56 +0300
http://bitbucket.org/pypy/pypy/changeset/873173b17a5e/

Log:	add how-to-update

diff --git a/lib-python/stdlib-upgrade.txt b/lib-python/stdlib-upgrade.txt
new file mode 100644
--- /dev/null
+++ b/lib-python/stdlib-upgrade.txt
@@ -0,0 +1,37 @@
+Process for upgrading the stdlib to a new cpython version
+==========================================================
+
+.. note::
+
+    overly detailed
+
+The idea is to 
+- exactly copy cpython's stdlib to a "vendor" branch on top of the previous
+  version
+- clean up the hg history for moved files so the merge from pypy will work
+- branch off that with a new integration branch
+- merge default or py3 into that branch, which will update with all the
+  modifications pypy made to the stdlib
+
+And in more detail:
+
+0. make sure your working dir is clean
+1. check out the branch vendor/stdlib (for 2.7) or vendor/stdlib-3-* (for py3k)
+   or create branch vendor/stdlib-3-*
+2. upgrade the files there
+   2a. remove lib-python/2.7/ or lib-python/3/
+   2b. copy the files from the cpython repo
+   2c. hg add lib-python/2.7/ or lib-python/3/
+   2d. hg remove --after
+   2e. show copied files in cpython repo by running `hg diff --git -r v<old> -r v<new> Lib | grep '^copy \(from\|to\)'` or `git diff --compact-summary v<old>..v<new> Lib` and search for `=>`
+   2f. fix copies / renames manually by running `hg copy --after <from> <to>` for each copied file
+3. update stdlib-version.txt with the output of hg -id from the cpython repo
+4. commit
+5. update to default / py3k
+6. create a integration branch for the new stdlib
+   (just hg branch stdlib-$version)
+7. merge vendor/stdlib or vendor/stdlib-3-*
+8. commit
+10. fix issues
+11. commit --close-branch
+12. merge to default / py3k

From pypy.commits at gmail.com  Mon May 20 07:30:38 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 20 May 2019 04:30:38 -0700 (PDT)
Subject: [pypy-commit] pypy vendor/stdlib: update to v2.7.16
Message-ID: <5ce28fde.1c69fb81.468d8.c83d@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: vendor/stdlib
Changeset: r96641:1779edf5c1e3
Date: 2019-05-20 12:55 +0300
http://bitbucket.org/pypy/pypy/changeset/1779edf5c1e3/

Log:	update to v2.7.16

diff too long, truncating to 2000 out of 46268 lines

diff --git a/lib-python/2.7/HTMLParser.py b/lib-python/2.7/HTMLParser.py
--- a/lib-python/2.7/HTMLParser.py
+++ b/lib-python/2.7/HTMLParser.py
@@ -462,11 +462,12 @@
             else:
                 # Cannot use name2codepoint directly, because HTMLParser supports apos,
                 # which is not part of HTML 4
-                import htmlentitydefs
                 if HTMLParser.entitydefs is None:
-                    entitydefs = HTMLParser.entitydefs = {'apos':u"'"}
+                    import htmlentitydefs
+                    entitydefs = {'apos':u"'"}
                     for k, v in htmlentitydefs.name2codepoint.iteritems():
                         entitydefs[k] = unichr(v)
+                    HTMLParser.entitydefs = entitydefs
                 try:
                     return self.entitydefs[s]
                 except KeyError:
diff --git a/lib-python/2.7/SocketServer.py b/lib-python/2.7/SocketServer.py
--- a/lib-python/2.7/SocketServer.py
+++ b/lib-python/2.7/SocketServer.py
@@ -229,6 +229,9 @@
                 # shutdown request and wastes cpu at all other times.
                 r, w, e = _eintr_retry(select.select, [self], [], [],
                                        poll_interval)
+                # bpo-35017: shutdown() called during select(), exit immediately.
+                if self.__shutdown_request:
+                    break
                 if self in r:
                     self._handle_request_noblock()
         finally:
diff --git a/lib-python/2.7/_pyio.py b/lib-python/2.7/_pyio.py
--- a/lib-python/2.7/_pyio.py
+++ b/lib-python/2.7/_pyio.py
@@ -1619,6 +1619,7 @@
         self.buffer.write(b)
         if self._line_buffering and (haslf or "\r" in s):
             self.flush()
+        self._set_decoded_chars('')
         self._snapshot = None
         if self._decoder:
             self._decoder.reset()
diff --git a/lib-python/2.7/_strptime.py b/lib-python/2.7/_strptime.py
--- a/lib-python/2.7/_strptime.py
+++ b/lib-python/2.7/_strptime.py
@@ -254,8 +254,8 @@
         # format directives (%m, etc.).
         regex_chars = re_compile(r"([\\.^$*+?\(\){}\[\]|])")
         format = regex_chars.sub(r"\\\1", format)
-        whitespace_replacement = re_compile('\s+')
-        format = whitespace_replacement.sub('\s+', format)
+        whitespace_replacement = re_compile(r'\s+')
+        format = whitespace_replacement.sub(r'\\s+', format)
         while '%' in format:
             directive_index = format.index('%')+1
             processed_format = "%s%s%s" % (processed_format,
diff --git a/lib-python/2.7/_threading_local.py b/lib-python/2.7/_threading_local.py
--- a/lib-python/2.7/_threading_local.py
+++ b/lib-python/2.7/_threading_local.py
@@ -57,11 +57,7 @@
 
   >>> class MyLocal(local):
   ...     number = 2
-  ...     initialized = False
   ...     def __init__(self, **kw):
-  ...         if self.initialized:
-  ...             raise SystemError('__init__ called too many times')
-  ...         self.initialized = True
   ...         self.__dict__.update(kw)
   ...     def squared(self):
   ...         return self.number ** 2
@@ -98,7 +94,7 @@
   >>> thread.start()
   >>> thread.join()
   >>> log
-  [[('color', 'red'), ('initialized', True)], 11]
+  [[('color', 'red')], 11]
 
 without affecting this thread's data:
 
diff --git a/lib-python/2.7/aifc.py b/lib-python/2.7/aifc.py
--- a/lib-python/2.7/aifc.py
+++ b/lib-python/2.7/aifc.py
@@ -288,6 +288,8 @@
     # _ssnd_chunk -- instantiation of a chunk class for the SSND chunk
     # _framesize -- size of one frame in the file
 
+    _file = None  # Set here since __del__ checks it
+
     def initfp(self, file):
         self._version = 0
         self._decomp = None
@@ -306,6 +308,7 @@
         else:
             raise Error, 'not an AIFF or AIFF-C file'
         self._comm_chunk_read = 0
+        self._ssnd_chunk = None
         while 1:
             self._ssnd_seek_needed = 1
             try:
@@ -341,10 +344,16 @@
             self._decomp.SetParams(params)
 
     def __init__(self, f):
-        if type(f) == type(''):
+        if isinstance(f, basestring):
             f = __builtin__.open(f, 'rb')
-        # else, assume it is an open file object already
-        self.initfp(f)
+            try:
+                self.initfp(f)
+            except:
+                f.close()
+                raise
+        else:
+            # assume it is an open file object already
+            self.initfp(f)
 
     #
     # User visible methods.
@@ -562,8 +571,10 @@
     # _datalength -- the size of the audio samples written to the header
     # _datawritten -- the size of the audio samples actually written
 
+    _file = None  # Set here since __del__ checks it
+
     def __init__(self, f):
-        if type(f) == type(''):
+        if isinstance(f, basestring):
             filename = f
             f = __builtin__.open(f, 'wb')
         else:
diff --git a/lib-python/2.7/argparse.py b/lib-python/2.7/argparse.py
--- a/lib-python/2.7/argparse.py
+++ b/lib-python/2.7/argparse.py
@@ -324,7 +324,11 @@
             if len(prefix) + len(usage) > text_width:
 
                 # break usage into wrappable parts
-                part_regexp = r'\(.*?\)+|\[.*?\]+|\S+'
+                part_regexp = (
+                    r'\(.*?\)+(?=\s|$)|'
+                    r'\[.*?\]+(?=\s|$)|'
+                    r'\S+'
+                )
                 opt_usage = format(optionals, groups)
                 pos_usage = format(positionals, groups)
                 opt_parts = _re.findall(part_regexp, opt_usage)
diff --git a/lib-python/2.7/asynchat.py b/lib-python/2.7/asynchat.py
--- a/lib-python/2.7/asynchat.py
+++ b/lib-python/2.7/asynchat.py
@@ -133,7 +133,7 @@
                 # no terminator, collect it all
                 self.collect_incoming_data (self.ac_in_buffer)
                 self.ac_in_buffer = ''
-            elif isinstance(terminator, int) or isinstance(terminator, long):
+            elif isinstance(terminator, (int, long)):
                 # numeric terminator
                 n = terminator
                 if lb < n:
diff --git a/lib-python/2.7/asyncore.py b/lib-python/2.7/asyncore.py
--- a/lib-python/2.7/asyncore.py
+++ b/lib-python/2.7/asyncore.py
@@ -633,7 +633,11 @@
         write = send
 
         def close(self):
-            os.close(self.fd)
+            if self.fd < 0:
+                return
+            fd = self.fd
+            self.fd = -1
+            os.close(fd)
 
         def fileno(self):
             return self.fd
diff --git a/lib-python/2.7/bsddb/test/test_associate.py b/lib-python/2.7/bsddb/test/test_associate.py
--- a/lib-python/2.7/bsddb/test/test_associate.py
+++ b/lib-python/2.7/bsddb/test/test_associate.py
@@ -114,6 +114,22 @@
             dupDB.close()
             self.fail("DBError exception was expected")
 
+    @unittest.skipUnless(db.version() >= (4, 6), 'Needs 4.6+')
+    def test_associateListError(self):
+        db1 = db.DB(self.env)
+        db1.open('bad.db', "a.db", db.DB_BTREE, db.DB_CREATE)
+        db2 = db.DB(self.env)
+        db2.open('bad.db', "b.db", db.DB_BTREE, db.DB_CREATE)
+
+        db1.associate(db2, lambda a, b: [0])
+
+        msg = "TypeError: The list returned by DB->associate callback" \
+              " should be a list of strings."
+        with test_support.captured_output("stderr") as s:
+            db1.put("0", "1")
+        db1.close()
+        db2.close()
+        self.assertEquals(s.getvalue().strip(), msg)
 
 
 #----------------------------------------------------------------------
@@ -233,7 +249,7 @@
         self.assertEqual(vals, None, vals)
 
         vals = secDB.pget('Unknown', txn=txn)
-        self.assertTrue(vals[0] == 99 or vals[0] == '99', vals)
+        self.assertIn(vals[0], (99, '99'), vals)
         vals[1].index('Unknown')
         vals[1].index('Unnamed')
         vals[1].index('unknown')
@@ -247,7 +263,8 @@
             if type(self.keytype) == type(''):
                 self.assertTrue(int(rec[0]))  # for primary db, key is a number
             else:
-                self.assertTrue(rec[0] and type(rec[0]) == type(0))
+                self.assertTrue(rec[0])
+                self.assertIs(type(rec[0]), int)
             count = count + 1
             if verbose:
                 print rec
@@ -262,7 +279,7 @@
 
         # test cursor pget
         vals = self.cur.pget('Unknown', flags=db.DB_LAST)
-        self.assertTrue(vals[1] == 99 or vals[1] == '99', vals)
+        self.assertIn(vals[1], (99, '99'), vals)
         self.assertEqual(vals[0], 'Unknown')
         vals[2].index('Unknown')
         vals[2].index('Unnamed')
diff --git a/lib-python/2.7/bsddb/test/test_basics.py b/lib-python/2.7/bsddb/test/test_basics.py
--- a/lib-python/2.7/bsddb/test/test_basics.py
+++ b/lib-python/2.7/bsddb/test/test_basics.py
@@ -597,7 +597,7 @@
 
         d.put("abcde", "ABCDE");
         num = d.truncate()
-        self.assertTrue(num >= 1, "truncate returned <= 0 on non-empty database")
+        self.assertGreaterEqual(num, 1, "truncate returned <= 0 on non-empty database")
         num = d.truncate()
         self.assertEqual(num, 0,
                 "truncate on empty DB returned nonzero (%r)" % (num,))
@@ -616,9 +616,9 @@
     if db.version() >= (4, 6):
         def test08_exists(self) :
             self.d.put("abcde", "ABCDE")
-            self.assertTrue(self.d.exists("abcde") == True,
+            self.assertEqual(self.d.exists("abcde"), True,
                     "DB->exists() returns wrong value")
-            self.assertTrue(self.d.exists("x") == False,
+            self.assertEqual(self.d.exists("x"), False,
                     "DB->exists() returns wrong value")
 
     #----------------------------------------
@@ -773,7 +773,7 @@
             if verbose:
                 print 'log file: ' + log
             logs = self.env.log_archive(db.DB_ARCH_REMOVE)
-            self.assertTrue(not logs)
+            self.assertFalse(logs)
 
         self.txn = self.env.txn_begin()
 
@@ -785,9 +785,9 @@
             self.d.put("abcde", "ABCDE", txn=txn)
             txn.commit()
             txn = self.env.txn_begin()
-            self.assertTrue(self.d.exists("abcde", txn=txn) == True,
+            self.assertEqual(self.d.exists("abcde", txn=txn), True,
                     "DB->exists() returns wrong value")
-            self.assertTrue(self.d.exists("x", txn=txn) == False,
+            self.assertEqual(self.d.exists("x", txn=txn), False,
                     "DB->exists() returns wrong value")
             txn.abort()
 
@@ -802,7 +802,7 @@
         d.put("abcde", "ABCDE");
         txn = self.env.txn_begin()
         num = d.truncate(txn)
-        self.assertTrue(num >= 1, "truncate returned <= 0 on non-empty database")
+        self.assertGreaterEqual(num, 1, "truncate returned <= 0 on non-empty database")
         num = d.truncate(txn)
         self.assertEqual(num, 0,
                 "truncate on empty DB returned nonzero (%r)" % (num,))
@@ -1086,7 +1086,7 @@
         a = "example of private object"
         self.obj.set_private(a)
         b = self.obj.get_private()
-        self.assertTrue(a is b)  # Object identity
+        self.assertIs(a, b)  # Object identity
 
     def test03_leak_assignment(self) :
         a = "example of private object"
diff --git a/lib-python/2.7/bsddb/test/test_dbenv.py b/lib-python/2.7/bsddb/test/test_dbenv.py
--- a/lib-python/2.7/bsddb/test/test_dbenv.py
+++ b/lib-python/2.7/bsddb/test/test_dbenv.py
@@ -54,15 +54,15 @@
                 self.env.set_cache_max(0, size)
                 size2 = self.env.get_cache_max()
                 self.assertEqual(0, size2[0])
-                self.assertTrue(size <= size2[1])
-                self.assertTrue(2*size > size2[1])
+                self.assertLessEqual(size, size2[1])
+                self.assertGreater(2*size, size2[1])
 
     if db.version() >= (4, 4) :
         def test_mutex_stat(self) :
             self.env.open(self.homeDir, db.DB_CREATE | db.DB_INIT_MPOOL |
                     db.DB_INIT_LOCK)
             stat = self.env.mutex_stat()
-            self.assertTrue("mutex_inuse_max" in stat)
+            self.assertIn("mutex_inuse_max", stat)
 
         def test_lg_filemode(self) :
             for i in [0600, 0660, 0666] :
@@ -128,8 +128,8 @@
                 i = i*1024*1024
                 self.env.set_lg_regionmax(i)
                 j = self.env.get_lg_regionmax()
-                self.assertTrue(i <= j)
-                self.assertTrue(2*i > j)
+                self.assertLessEqual(i, j)
+                self.assertGreater(2*i, j)
 
         def test_lk_detect(self) :
             flags= [db.DB_LOCK_DEFAULT, db.DB_LOCK_EXPIRE, db.DB_LOCK_MAXLOCKS,
@@ -150,10 +150,10 @@
         def test_lg_bsize(self) :
             log_size = 70*1024
             self.env.set_lg_bsize(log_size)
-            self.assertTrue(self.env.get_lg_bsize() >= log_size)
-            self.assertTrue(self.env.get_lg_bsize() < 4*log_size)
+            self.assertGreaterEqual(self.env.get_lg_bsize(), log_size)
+            self.assertLess(self.env.get_lg_bsize(), 4*log_size)
             self.env.set_lg_bsize(4*log_size)
-            self.assertTrue(self.env.get_lg_bsize() >= 4*log_size)
+            self.assertGreaterEqual(self.env.get_lg_bsize(), 4*log_size)
 
         def test_setget_data_dirs(self) :
             dirs = ("a", "b", "c", "d")
@@ -185,7 +185,7 @@
             self.assertEqual(cachesize2[0], cachesize3[0])
             self.assertEqual(cachesize2[2], cachesize3[2])
             # In Berkeley DB 5.1, the cachesize can change when opening the Env
-            self.assertTrue(cachesize2[1] <= cachesize3[1])
+            self.assertLessEqual(cachesize2[1], cachesize3[1])
 
         def test_set_cachesize_dbenv_db(self) :
             # You can not configure the cachesize using
@@ -299,7 +299,7 @@
             msg = "This is a test..."
             self.env.log_printf(msg)
             logc = self.env.log_cursor()
-            self.assertTrue(msg in (logc.last()[1]))
+            self.assertIn(msg, logc.last()[1])
 
     if db.version() >= (4, 7) :
         def test_log_config(self) :
@@ -341,21 +341,21 @@
             txn.commit()
             logc = self.env.log_cursor()
             logc.last()  # Skip the commit
-            self.assertTrue(msg in (logc.prev()[1]))
+            self.assertIn(msg, logc.prev()[1])
 
             msg = "This is another test..."
             txn = self.env.txn_begin()
             self.env.log_printf(msg, txn=txn)
             txn.abort()  # Do not store the new message
             logc.last()  # Skip the abort
-            self.assertTrue(msg not in (logc.prev()[1]))
+            self.assertNotIn(msg, logc.prev()[1])
 
             msg = "This is a third test..."
             txn = self.env.txn_begin()
             self.env.log_printf(msg, txn=txn)
             txn.commit()  # Do not store the new message
             logc.last()  # Skip the commit
-            self.assertTrue(msg in (logc.prev()[1]))
+            self.assertIn(msg, logc.prev()[1])
 
 
 class DBEnv_memp(DBEnv):
@@ -372,39 +372,39 @@
 
     def test_memp_1_trickle(self) :
         self.db.put("hi", "bye")
-        self.assertTrue(self.env.memp_trickle(100) > 0)
+        self.assertGreater(self.env.memp_trickle(100), 0)
 
 # Preserve the order, do "memp_trickle" test first
     def test_memp_2_sync(self) :
         self.db.put("hi", "bye")
         self.env.memp_sync()  # Full flush
         # Nothing to do...
-        self.assertTrue(self.env.memp_trickle(100) == 0)
+        self.assertEqual(self.env.memp_trickle(100), 0)
 
         self.db.put("hi", "bye2")
         self.env.memp_sync((1, 0))  # NOP, probably
         # Something to do... or not
-        self.assertTrue(self.env.memp_trickle(100) >= 0)
+        self.assertGreaterEqual(self.env.memp_trickle(100), 0)
 
         self.db.put("hi", "bye3")
         self.env.memp_sync((123, 99))  # Full flush
         # Nothing to do...
-        self.assertTrue(self.env.memp_trickle(100) == 0)
+        self.assertEqual(self.env.memp_trickle(100), 0)
 
     def test_memp_stat_1(self) :
         stats = self.env.memp_stat()  # No param
-        self.assertTrue(len(stats)==2)
-        self.assertTrue("cache_miss" in stats[0])
+        self.assertEqual(len(stats), 2)
+        self.assertIn("cache_miss", stats[0])
         stats = self.env.memp_stat(db.DB_STAT_CLEAR)  # Positional param
-        self.assertTrue("cache_miss" in stats[0])
+        self.assertIn("cache_miss", stats[0])
         stats = self.env.memp_stat(flags=0)  # Keyword param
-        self.assertTrue("cache_miss" in stats[0])
+        self.assertIn("cache_miss", stats[0])
 
     def test_memp_stat_2(self) :
         stats=self.env.memp_stat()[1]
-        self.assertTrue(len(stats))==1
-        self.assertTrue("test" in stats)
-        self.assertTrue("page_in" in stats["test"])
+        self.assertEqual(len(stats), 1)
+        self.assertIn("test", stats)
+        self.assertIn("page_in", stats["test"])
 
 class DBEnv_logcursor(DBEnv):
     def setUp(self):
@@ -426,28 +426,28 @@
         DBEnv.tearDown(self)
 
     def _check_return(self, value) :
-        self.assertTrue(isinstance(value, tuple))
+        self.assertIsInstance(value, tuple)
         self.assertEqual(len(value), 2)
-        self.assertTrue(isinstance(value[0], tuple))
+        self.assertIsInstance(value[0], tuple)
         self.assertEqual(len(value[0]), 2)
-        self.assertTrue(isinstance(value[0][0], int))
-        self.assertTrue(isinstance(value[0][1], int))
-        self.assertTrue(isinstance(value[1], str))
+        self.assertIsInstance(value[0][0], int)
+        self.assertIsInstance(value[0][1], int)
+        self.assertIsInstance(value[1], str)
 
     # Preserve test order
     def test_1_first(self) :
         logc = self.env.log_cursor()
         v = logc.first()
         self._check_return(v)
-        self.assertTrue((1, 1) < v[0])
-        self.assertTrue(len(v[1])>0)
+        self.assertLess((1, 1), v[0])
+        self.assertGreater(len(v[1]), 0)
 
     def test_2_last(self) :
         logc = self.env.log_cursor()
         lsn_first = logc.first()[0]
         v = logc.last()
         self._check_return(v)
-        self.assertTrue(lsn_first < v[0])
+        self.assertLess(lsn_first, v[0])
 
     def test_3_next(self) :
         logc = self.env.log_cursor()
@@ -456,16 +456,16 @@
         lsn_first = logc.first()[0]
         v = logc.next()
         self._check_return(v)
-        self.assertTrue(lsn_first < v[0])
-        self.assertTrue(lsn_last > v[0])
+        self.assertLess(lsn_first, v[0])
+        self.assertGreater(lsn_last, v[0])
 
         v2 = logc.next()
-        self.assertTrue(v2[0] > v[0])
-        self.assertTrue(lsn_last > v2[0])
+        self.assertGreater(v2[0], v[0])
+        self.assertGreater(lsn_last, v2[0])
 
         v3 = logc.next()
-        self.assertTrue(v3[0] > v2[0])
-        self.assertTrue(lsn_last > v3[0])
+        self.assertGreater(v3[0], v2[0])
+        self.assertGreater(lsn_last, v3[0])
 
     def test_4_prev(self) :
         logc = self.env.log_cursor()
@@ -474,16 +474,16 @@
         lsn_last = logc.last()[0]
         v = logc.prev()
         self._check_return(v)
-        self.assertTrue(lsn_first < v[0])
-        self.assertTrue(lsn_last > v[0])
+        self.assertLess(lsn_first, v[0])
+        self.assertGreater(lsn_last, v[0])
 
         v2 = logc.prev()
-        self.assertTrue(v2[0] < v[0])
-        self.assertTrue(lsn_first < v2[0])
+        self.assertLess(v2[0], v[0])
+        self.assertLess(lsn_first, v2[0])
 
         v3 = logc.prev()
-        self.assertTrue(v3[0] < v2[0])
-        self.assertTrue(lsn_first < v3[0])
+        self.assertLess(v3[0], v2[0])
+        self.assertLess(lsn_first, v3[0])
 
     def test_5_current(self) :
         logc = self.env.log_cursor()
diff --git a/lib-python/2.7/bsddb/test/test_dbshelve.py b/lib-python/2.7/bsddb/test/test_dbshelve.py
--- a/lib-python/2.7/bsddb/test/test_dbshelve.py
+++ b/lib-python/2.7/bsddb/test/test_dbshelve.py
@@ -248,7 +248,7 @@
             self.assertEqual(value.L, [x] * 10)
 
         else:
-            self.assertTrue(0, 'Unknown key type, fix the test')
+            self.fail('Unknown key type, fix the test')
 
 #----------------------------------------------------------------------
 
diff --git a/lib-python/2.7/bsddb/test/test_dbtables.py b/lib-python/2.7/bsddb/test/test_dbtables.py
--- a/lib-python/2.7/bsddb/test/test_dbtables.py
+++ b/lib-python/2.7/bsddb/test/test_dbtables.py
@@ -82,8 +82,8 @@
             colval = pickle.loads(values[0][colname])
         else :
             colval = pickle.loads(bytes(values[0][colname], "iso8859-1"))
-        self.assertTrue(colval > 3.141)
-        self.assertTrue(colval < 3.142)
+        self.assertGreater(colval, 3.141)
+        self.assertLess(colval, 3.142)
 
 
     def test02(self):
diff --git a/lib-python/2.7/bsddb/test/test_distributed_transactions.py b/lib-python/2.7/bsddb/test/test_distributed_transactions.py
--- a/lib-python/2.7/bsddb/test/test_distributed_transactions.py
+++ b/lib-python/2.7/bsddb/test/test_distributed_transactions.py
@@ -79,7 +79,7 @@
         recovered_txns=self.dbenv.txn_recover()
         self.assertEqual(self.num_txns,len(recovered_txns))
         for gid,txn in recovered_txns :
-            self.assertTrue(gid in txns)
+            self.assertIn(gid, txns)
         del txn
         del recovered_txns
 
@@ -122,7 +122,7 @@
     # Be sure there are not pending transactions.
     # Check also database size.
         recovered_txns=self.dbenv.txn_recover()
-        self.assertTrue(len(recovered_txns)==0)
+        self.assertEqual(len(recovered_txns), 0)
         self.assertEqual(len(committed_txns),self.db.stat()["nkeys"])
 
 class DBTxn_distributedSYNC(DBTxn_distributed):
diff --git a/lib-python/2.7/bsddb/test/test_lock.py b/lib-python/2.7/bsddb/test/test_lock.py
--- a/lib-python/2.7/bsddb/test/test_lock.py
+++ b/lib-python/2.7/bsddb/test/test_lock.py
@@ -2,6 +2,7 @@
 TestCases for testing the locking sub-system.
 """
 
+import sys
 import time
 
 import unittest
@@ -10,7 +11,6 @@
 
 if have_threads :
     from threading import Thread
-    import sys
     if sys.version_info[0] < 3 :
         from threading import currentThread
     else :
@@ -129,7 +129,14 @@
         end_time=time.time()
         deadlock_detection.end=True
         # Floating point rounding
-        self.assertTrue((end_time-start_time) >= 0.0999)
+        if sys.platform == 'win32':
+            # bpo-30850: On Windows, tolerate 50 ms whereas 100 ms is expected.
+            # The lock sometimes times out after only 58 ms. Windows clocks
+            # have a bad resolution and bad accuracy.
+            min_dt = 0.050
+        else:
+            min_dt = 0.0999
+        self.assertGreaterEqual(end_time-start_time, min_dt)
         self.env.lock_put(lock)
         t.join()
 
@@ -137,7 +144,7 @@
         self.env.lock_id_free(anID2)
 
         if db.version() >= (4,6):
-            self.assertTrue(deadlock_detection.count>0)
+            self.assertGreater(deadlock_detection.count, 0)
 
     def theThread(self, lockType):
         import sys
diff --git a/lib-python/2.7/bsddb/test/test_misc.py b/lib-python/2.7/bsddb/test/test_misc.py
--- a/lib-python/2.7/bsddb/test/test_misc.py
+++ b/lib-python/2.7/bsddb/test/test_misc.py
@@ -25,7 +25,7 @@
     def test02_db_home(self):
         env = db.DBEnv()
         # check for crash fixed when db_home is used before open()
-        self.assertTrue(env.db_home is None)
+        self.assertIsNone(env.db_home)
         env.open(self.homeDir, db.DB_CREATE)
         if sys.version_info[0] < 3 :
             self.assertEqual(self.homeDir, env.db_home)
diff --git a/lib-python/2.7/bsddb/test/test_recno.py b/lib-python/2.7/bsddb/test/test_recno.py
--- a/lib-python/2.7/bsddb/test/test_recno.py
+++ b/lib-python/2.7/bsddb/test/test_recno.py
@@ -18,7 +18,7 @@
         def assertIsInstance(self, obj, datatype, msg=None) :
             return self.assertEqual(type(obj), datatype, msg=msg)
         def assertGreaterEqual(self, a, b, msg=None) :
-            return self.assertTrue(a>=b, msg=msg)
+            return self.assertGreaterEqual(a, b, msg=msg)
 
 
     def setUp(self):
diff --git a/lib-python/2.7/bsddb/test/test_replication.py b/lib-python/2.7/bsddb/test/test_replication.py
--- a/lib-python/2.7/bsddb/test/test_replication.py
+++ b/lib-python/2.7/bsddb/test/test_replication.py
@@ -186,20 +186,18 @@
         d = d.values()[0]  # There is only one
         self.assertEqual(d[0], "127.0.0.1")
         self.assertEqual(d[1], client_port)
-        self.assertTrue((d[2]==db.DB_REPMGR_CONNECTED) or \
-                (d[2]==db.DB_REPMGR_DISCONNECTED))
+        self.assertIn(d[2], (db.DB_REPMGR_CONNECTED, db.DB_REPMGR_DISCONNECTED))
 
         d = self.dbenvClient.repmgr_site_list()
         self.assertEqual(len(d), 1)
         d = d.values()[0]  # There is only one
         self.assertEqual(d[0], "127.0.0.1")
         self.assertEqual(d[1], master_port)
-        self.assertTrue((d[2]==db.DB_REPMGR_CONNECTED) or \
-                (d[2]==db.DB_REPMGR_DISCONNECTED))
+        self.assertIn(d[2], (db.DB_REPMGR_CONNECTED, db.DB_REPMGR_DISCONNECTED))
 
         if db.version() >= (4,6) :
             d = self.dbenvMaster.repmgr_stat(flags=db.DB_STAT_CLEAR);
-            self.assertTrue("msgs_queued" in d)
+            self.assertIn("msgs_queued", d)
 
         self.dbMaster=db.DB(self.dbenvMaster)
         txn=self.dbenvMaster.txn_begin()
@@ -247,7 +245,7 @@
         if time.time()>=timeout and startup_timeout:
             self.skipTest("replication test skipped due to random failure, "
                 "see issue 3892")
-        self.assertTrue(time.time()<timeout)
+        self.assertLess(time.time(), timeout)
         self.assertEqual("123", v)
 
         txn=self.dbenvMaster.txn_begin()
@@ -260,7 +258,7 @@
             txn.commit()
             if v is None :
                 time.sleep(0.02)
-        self.assertTrue(time.time()<timeout)
+        self.assertLess(time.time(), timeout)
         self.assertEqual(None, v)
 
 class DBBaseReplication(DBReplication) :
@@ -381,7 +379,7 @@
         while (time.time()<timeout) and not (self.confirmed_master and
                 self.client_startupdone) :
             time.sleep(0.02)
-        self.assertTrue(time.time()<timeout)
+        self.assertLess(time.time(), timeout)
 
         self.dbMaster=db.DB(self.dbenvMaster)
         txn=self.dbenvMaster.txn_begin()
@@ -410,7 +408,7 @@
             break
 
         d = self.dbenvMaster.rep_stat(flags=db.DB_STAT_CLEAR);
-        self.assertTrue("master_changes" in d)
+        self.assertIn("master_changes", d)
 
         txn=self.dbenvMaster.txn_begin()
         self.dbMaster.put("ABC", "123", txn=txn)
@@ -424,7 +422,7 @@
             txn.commit()
             if v is None :
                 time.sleep(0.02)
-        self.assertTrue(time.time()<timeout)
+        self.assertLess(time.time(), timeout)
         self.assertEqual("123", v)
 
         txn=self.dbenvMaster.txn_begin()
@@ -437,7 +435,7 @@
             txn.commit()
             if v is None :
                 time.sleep(0.02)
-        self.assertTrue(time.time()<timeout)
+        self.assertLess(time.time(), timeout)
         self.assertEqual(None, v)
 
     if db.version() >= (4,7) :
diff --git a/lib-python/2.7/bsddb/test/test_sequence.py b/lib-python/2.7/bsddb/test/test_sequence.py
--- a/lib-python/2.7/bsddb/test/test_sequence.py
+++ b/lib-python/2.7/bsddb/test/test_sequence.py
@@ -82,7 +82,7 @@
         stat = self.seq.stat()
         for param in ('nowait', 'min', 'max', 'value', 'current',
                       'flags', 'cache_size', 'last_value', 'wait'):
-            self.assertTrue(param in stat, "parameter %s isn't in stat info" % param)
+            self.assertIn(param, stat, "parameter %s isn't in stat info" % param)
 
     if db.version() >= (4,7) :
         # This code checks a crash solved in Berkeley DB 4.7
diff --git a/lib-python/2.7/bsddb/test/test_thread.py b/lib-python/2.7/bsddb/test/test_thread.py
--- a/lib-python/2.7/bsddb/test/test_thread.py
+++ b/lib-python/2.7/bsddb/test/test_thread.py
@@ -85,7 +85,7 @@
         readers_per_writer=self.readers//self.writers
         self.assertEqual(self.records,self.writers*records_per_writer)
         self.assertEqual(self.readers,self.writers*readers_per_writer)
-        self.assertTrue((records_per_writer%readers_per_writer)==0)
+        self.assertEqual(records_per_writer%readers_per_writer, 0)
         readers = []
 
         for x in xrange(self.readers):
@@ -213,7 +213,7 @@
         readers_per_writer=self.readers//self.writers
         self.assertEqual(self.records,self.writers*records_per_writer)
         self.assertEqual(self.readers,self.writers*readers_per_writer)
-        self.assertTrue((records_per_writer%readers_per_writer)==0)
+        self.assertEqual(records_per_writer%readers_per_writer, 0)
 
         readers = []
         for x in xrange(self.readers):
@@ -339,7 +339,7 @@
         readers_per_writer=self.readers//self.writers
         self.assertEqual(self.records,self.writers*records_per_writer)
         self.assertEqual(self.readers,self.writers*readers_per_writer)
-        self.assertTrue((records_per_writer%readers_per_writer)==0)
+        self.assertEqual(records_per_writer%readers_per_writer, 0)
 
         readers=[]
         for x in xrange(self.readers):
diff --git a/lib-python/2.7/cProfile.py b/lib-python/2.7/cProfile.py
--- a/lib-python/2.7/cProfile.py
+++ b/lib-python/2.7/cProfile.py
@@ -64,11 +64,11 @@
 # ____________________________________________________________
 
 class Profile(_lsprof.Profiler):
-    """Profile(custom_timer=None, time_unit=None, subcalls=True, builtins=True)
+    """Profile(timer=None, timeunit=None, subcalls=True, builtins=True)
 
     Builds a profiler object using the specified timer function.
     The default timer is a fast built-in one based on real time.
-    For custom timer functions returning integers, time_unit can
+    For custom timer functions returning integers, timeunit can
     be a float specifying a scale (i.e. how long each integer unit
     is, in seconds).
     """
@@ -161,7 +161,7 @@
 # ____________________________________________________________
 
 def main():
-    import os, sys
+    import os, sys, pstats
     from optparse import OptionParser
     usage = "cProfile.py [-o output_file_path] [-s sort] scriptfile [arg] ..."
     parser = OptionParser(usage=usage)
@@ -170,7 +170,8 @@
         help="Save stats to <outfile>", default=None)
     parser.add_option('-s', '--sort', dest="sort",
         help="Sort order when printing to stdout, based on pstats.Stats class",
-        default=-1)
+        default=-1,
+        choices=sorted(pstats.Stats.sort_arg_dict_default))
 
     if not sys.argv[1:]:
         parser.print_usage()
diff --git a/lib-python/2.7/cgi.py b/lib-python/2.7/cgi.py
--- a/lib-python/2.7/cgi.py
+++ b/lib-python/2.7/cgi.py
@@ -184,11 +184,12 @@
     return urlparse.parse_qs(qs, keep_blank_values, strict_parsing)
 
 
-def parse_qsl(qs, keep_blank_values=0, strict_parsing=0):
+def parse_qsl(qs, keep_blank_values=0, strict_parsing=0, max_num_fields=None):
     """Parse a query given as a string argument."""
     warn("cgi.parse_qsl is deprecated, use urlparse.parse_qsl instead",
          PendingDeprecationWarning, 2)
-    return urlparse.parse_qsl(qs, keep_blank_values, strict_parsing)
+    return urlparse.parse_qsl(qs, keep_blank_values, strict_parsing,
+                              max_num_fields)
 
 def parse_multipart(fp, pdict):
     """Parse multipart input.
@@ -393,7 +394,8 @@
     """
 
     def __init__(self, fp=None, headers=None, outerboundary="",
-                 environ=os.environ, keep_blank_values=0, strict_parsing=0):
+                 environ=os.environ, keep_blank_values=0, strict_parsing=0,
+                 max_num_fields=None):
         """Constructor.  Read multipart/* until last part.
 
         Arguments, all optional:
@@ -420,10 +422,14 @@
             If false (the default), errors are silently ignored.
             If true, errors raise a ValueError exception.
 
+        max_num_fields: int. If set, then __init__ throws a ValueError
+            if there are more than n fields read by parse_qsl().
+
         """
         method = 'GET'
         self.keep_blank_values = keep_blank_values
         self.strict_parsing = strict_parsing
+        self.max_num_fields = max_num_fields
         if 'REQUEST_METHOD' in environ:
             method = environ['REQUEST_METHOD'].upper()
         self.qs_on_post = None
@@ -606,10 +612,9 @@
         qs = self.fp.read(self.length)
         if self.qs_on_post:
             qs += '&' + self.qs_on_post
-        self.list = list = []
-        for key, value in urlparse.parse_qsl(qs, self.keep_blank_values,
-                                            self.strict_parsing):
-            list.append(MiniFieldStorage(key, value))
+        query = urlparse.parse_qsl(qs, self.keep_blank_values,
+                                   self.strict_parsing, self.max_num_fields)
+        self.list = [MiniFieldStorage(key, value) for key, value in query]
         self.skip_lines()
 
     FieldStorageClass = None
@@ -621,19 +626,38 @@
             raise ValueError, 'Invalid boundary in multipart form: %r' % (ib,)
         self.list = []
         if self.qs_on_post:
-            for key, value in urlparse.parse_qsl(self.qs_on_post,
-                                self.keep_blank_values, self.strict_parsing):
-                self.list.append(MiniFieldStorage(key, value))
+            query = urlparse.parse_qsl(self.qs_on_post,
+                                       self.keep_blank_values,
+                                       self.strict_parsing,
+                                       self.max_num_fields)
+            self.list.extend(MiniFieldStorage(key, value)
+                             for key, value in query)
             FieldStorageClass = None
 
+        # Propagate max_num_fields into the sub class appropriately
+        max_num_fields = self.max_num_fields
+        if max_num_fields is not None:
+            max_num_fields -= len(self.list)
+
         klass = self.FieldStorageClass or self.__class__
         part = klass(self.fp, {}, ib,
-                     environ, keep_blank_values, strict_parsing)
+                     environ, keep_blank_values, strict_parsing,
+                     max_num_fields)
+
         # Throw first part away
         while not part.done:
             headers = rfc822.Message(self.fp)
             part = klass(self.fp, headers, ib,
-                         environ, keep_blank_values, strict_parsing)
+                         environ, keep_blank_values, strict_parsing,
+                         max_num_fields)
+
+            if max_num_fields is not None:
+                max_num_fields -= 1
+                if part.list:
+                    max_num_fields -= len(part.list)
+                if max_num_fields < 0:
+                    raise ValueError('Max number of fields exceeded')
+
             self.list.append(part)
         self.skip_lines()
 
diff --git a/lib-python/2.7/cgitb.py b/lib-python/2.7/cgitb.py
--- a/lib-python/2.7/cgitb.py
+++ b/lib-python/2.7/cgitb.py
@@ -125,7 +125,7 @@
         args, varargs, varkw, locals = inspect.getargvalues(frame)
         call = ''
         if func != '?':
-            call = 'in ' + strong(func) + \
+            call = 'in ' + strong(pydoc.html.escape(func)) + \
                 inspect.formatargvalues(args, varargs, varkw, locals,
                     formatvalue=lambda value: '=' + pydoc.html.repr(value))
 
@@ -285,7 +285,7 @@
 
         if self.display:
             if plain:
-                doc = doc.replace('&', '&amp;').replace('<', '&lt;')
+                doc = pydoc.html.escape(doc)
                 self.file.write('<pre>' + doc + '</pre>\n')
             else:
                 self.file.write(doc + '\n')
diff --git a/lib-python/2.7/codecs.py b/lib-python/2.7/codecs.py
--- a/lib-python/2.7/codecs.py
+++ b/lib-python/2.7/codecs.py
@@ -472,15 +472,17 @@
             self.charbuffer = "".join(self.linebuffer)
             self.linebuffer = None
 
+        if chars < 0:
+            # For compatibility with other read() methods that take a
+            # single argument
+            chars = size
+
         # read until we get the required number of characters (if available)
         while True:
             # can the request be satisfied from the character buffer?
             if chars >= 0:
                 if len(self.charbuffer) >= chars:
                     break
-            elif size >= 0:
-                if len(self.charbuffer) >= size:
-                    break
             # we need more data
             if size < 0:
                 newdata = self.stream.read()
diff --git a/lib-python/2.7/compiler/pyassem.py b/lib-python/2.7/compiler/pyassem.py
--- a/lib-python/2.7/compiler/pyassem.py
+++ b/lib-python/2.7/compiler/pyassem.py
@@ -581,7 +581,7 @@
 
 def twobyte(val):
     """Convert an int argument into high and low bytes"""
-    assert isinstance(val, int)
+    assert isinstance(val, (int, long))
     return divmod(val, 256)
 
 class LineAddrTable:
diff --git a/lib-python/2.7/compiler/transformer.py b/lib-python/2.7/compiler/transformer.py
--- a/lib-python/2.7/compiler/transformer.py
+++ b/lib-python/2.7/compiler/transformer.py
@@ -1526,7 +1526,7 @@
 def debug_tree(tree):
     l = []
     for elt in tree:
-        if isinstance(elt, int):
+        if isinstance(elt, (int, long)):
             l.append(_names.get(elt, elt))
         elif isinstance(elt, str):
             l.append(elt)
diff --git a/lib-python/2.7/copy_reg.py b/lib-python/2.7/copy_reg.py
--- a/lib-python/2.7/copy_reg.py
+++ b/lib-python/2.7/copy_reg.py
@@ -127,7 +127,11 @@
                         continue
                     # mangled names
                     elif name.startswith('__') and not name.endswith('__'):
-                        names.append('_%s%s' % (c.__name__, name))
+                        stripped = c.__name__.lstrip('_')
+                        if stripped:
+                            names.append('_%s%s' % (stripped, name))
+                        else:
+                            names.append(name)
                     else:
                         names.append(name)
 
diff --git a/lib-python/2.7/csv.py b/lib-python/2.7/csv.py
--- a/lib-python/2.7/csv.py
+++ b/lib-python/2.7/csv.py
@@ -217,7 +217,7 @@
         matches = []
         for restr in ('(?P<delim>[^\w\n"\'])(?P<space> ?)(?P<quote>["\']).*?(?P=quote)(?P=delim)', # ,".*?",
                       '(?:^|\n)(?P<quote>["\']).*?(?P=quote)(?P<delim>[^\w\n"\'])(?P<space> ?)',   #  ".*?",
-                      '(?P<delim>>[^\w\n"\'])(?P<space> ?)(?P<quote>["\']).*?(?P=quote)(?:$|\n)',  # ,".*?"
+                      '(?P<delim>[^\w\n"\'])(?P<space> ?)(?P<quote>["\']).*?(?P=quote)(?:$|\n)',   # ,".*?"
                       '(?:^|\n)(?P<quote>["\']).*?(?P=quote)(?:$|\n)'):                            #  ".*?" (no delim, no space)
             regexp = re.compile(restr, re.DOTALL | re.MULTILINE)
             matches = regexp.findall(data)
diff --git a/lib-python/2.7/ctypes/__init__.py b/lib-python/2.7/ctypes/__init__.py
--- a/lib-python/2.7/ctypes/__init__.py
+++ b/lib-python/2.7/ctypes/__init__.py
@@ -342,6 +342,10 @@
     """
     _func_flags_ = _FUNCFLAG_CDECL
     _func_restype_ = c_int
+    # default values for repr
+    _name = '<uninitialized>'
+    _handle = 0
+    _FuncPtr = None
 
     def __init__(self, name, mode=DEFAULT_MODE, handle=None,
                  use_errno=False,
diff --git a/lib-python/2.7/ctypes/test/test_anon.py b/lib-python/2.7/ctypes/test/test_anon.py
--- a/lib-python/2.7/ctypes/test/test_anon.py
+++ b/lib-python/2.7/ctypes/test/test_anon.py
@@ -1,4 +1,5 @@
 import unittest
+from test.support import cpython_only
 from ctypes import *
 
 class AnonTest(unittest.TestCase):
@@ -35,6 +36,18 @@
                                                       {"_fields_": [],
                                                        "_anonymous_": ["x"]}))
 
+    @cpython_only
+    def test_issue31490(self):
+        # There shouldn't be an assertion failure in case the class has an
+        # attribute whose name is specified in _anonymous_ but not in _fields_.
+
+        # AttributeError: 'x' is specified in _anonymous_ but not in _fields_
+        with self.assertRaises(AttributeError):
+            class Name(Structure):
+                _fields_ = []
+                _anonymous_ = ["x"]
+                x = 42
+
     def test_nested(self):
         class ANON_S(Structure):
             _fields_ = [("a", c_int)]
diff --git a/lib-python/2.7/ctypes/test/test_arrays.py b/lib-python/2.7/ctypes/test/test_arrays.py
--- a/lib-python/2.7/ctypes/test/test_arrays.py
+++ b/lib-python/2.7/ctypes/test/test_arrays.py
@@ -1,4 +1,6 @@
 import unittest
+from test.support import precisionbigmemtest, _2G
+import sys
 from ctypes import *
 
 from ctypes.test import need_symbol
@@ -132,5 +134,10 @@
         t2 = my_int * 1
         self.assertIs(t1, t2)
 
+    @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform')
+    @precisionbigmemtest(size=_2G, memuse=1, dry_run=False)
+    def test_large_array(self, size):
+        a = c_char * size
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_as_parameter.py b/lib-python/2.7/ctypes/test/test_as_parameter.py
--- a/lib-python/2.7/ctypes/test/test_as_parameter.py
+++ b/lib-python/2.7/ctypes/test/test_as_parameter.py
@@ -24,7 +24,7 @@
         f.argtypes = [c_byte, c_wchar, c_int, c_long, c_float, c_double]
         result = f(self.wrap(1), self.wrap(u"x"), self.wrap(3), self.wrap(4), self.wrap(5.0), self.wrap(6.0))
         self.assertEqual(result, 139)
-        self.assertTrue(type(result), int)
+        self.assertIs(type(result), int)
 
     def test_pointers(self):
         f = dll._testfunc_p_p
diff --git a/lib-python/2.7/ctypes/test/test_callbacks.py b/lib-python/2.7/ctypes/test/test_callbacks.py
--- a/lib-python/2.7/ctypes/test/test_callbacks.py
+++ b/lib-python/2.7/ctypes/test/test_callbacks.py
@@ -250,6 +250,7 @@
     def test_callback_large_struct(self):
         class Check: pass
 
+        # This should mirror the structure in Modules/_ctypes/_ctypes_test.c
         class X(Structure):
             _fields_ = [
                 ('first', c_ulong),
@@ -261,6 +262,11 @@
             check.first = s.first
             check.second = s.second
             check.third = s.third
+            # See issue #29565.
+            # The structure should be passed by value, so
+            # any changes to it should not be reflected in
+            # the value passed
+            s.first = s.second = s.third = 0x0badf00d
 
         check = Check()
         s = X()
@@ -281,6 +287,11 @@
         self.assertEqual(check.first, 0xdeadbeef)
         self.assertEqual(check.second, 0xcafebabe)
         self.assertEqual(check.third, 0x0bad1dea)
+        # See issue #29565.
+        # Ensure that the original struct is unchanged.
+        self.assertEqual(s.first, check.first)
+        self.assertEqual(s.second, check.second)
+        self.assertEqual(s.third, check.third)
 
 ################################################################
 
diff --git a/lib-python/2.7/ctypes/test/test_frombuffer.py b/lib-python/2.7/ctypes/test/test_frombuffer.py
--- a/lib-python/2.7/ctypes/test/test_frombuffer.py
+++ b/lib-python/2.7/ctypes/test/test_frombuffer.py
@@ -78,12 +78,21 @@
                           (c_int * 1).from_buffer_copy, a, 16 * sizeof(c_int))
 
     def test_abstract(self):
+        from ctypes import _Pointer, _SimpleCData, _CFuncPtr
+
         self.assertRaises(TypeError, Array.from_buffer, bytearray(10))
         self.assertRaises(TypeError, Structure.from_buffer, bytearray(10))
         self.assertRaises(TypeError, Union.from_buffer, bytearray(10))
+        self.assertRaises(TypeError, _CFuncPtr.from_buffer, bytearray(10))
+        self.assertRaises(TypeError, _Pointer.from_buffer, bytearray(10))
+        self.assertRaises(TypeError, _SimpleCData.from_buffer, bytearray(10))
+
         self.assertRaises(TypeError, Array.from_buffer_copy, b"123")
         self.assertRaises(TypeError, Structure.from_buffer_copy, b"123")
         self.assertRaises(TypeError, Union.from_buffer_copy, b"123")
+        self.assertRaises(TypeError, _CFuncPtr.from_buffer_copy, b"123")
+        self.assertRaises(TypeError, _Pointer.from_buffer_copy, b"123")
+        self.assertRaises(TypeError, _SimpleCData.from_buffer_copy, b"123")
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_funcptr.py b/lib-python/2.7/ctypes/test/test_funcptr.py
--- a/lib-python/2.7/ctypes/test/test_funcptr.py
+++ b/lib-python/2.7/ctypes/test/test_funcptr.py
@@ -123,5 +123,10 @@
         self.assertEqual(strtok(None, "\n"), "c")
         self.assertEqual(strtok(None, "\n"), None)
 
+    def test_abstract(self):
+        from ctypes import _CFuncPtr
+
+        self.assertRaises(TypeError, _CFuncPtr, 13, "name", 42, "iid")
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_loading.py b/lib-python/2.7/ctypes/test/test_loading.py
--- a/lib-python/2.7/ctypes/test/test_loading.py
+++ b/lib-python/2.7/ctypes/test/test_loading.py
@@ -3,6 +3,7 @@
 import os
 from ctypes.util import find_library
 from ctypes.test import is_resource_enabled
+import test.test_support as support
 
 libc_name = None
 if os.name == "nt":
@@ -27,6 +28,12 @@
         CDLL(os.path.basename(libc_name))
         self.assertRaises(OSError, CDLL, self.unknowndll)
 
+    @support.requires_unicode
+    @unittest.skipUnless(libc_name is not None, 'could not find libc')
+    def test_load_unicode(self):
+        CDLL(unicode(libc_name))
+        self.assertRaises(OSError, CDLL, unicode(self.unknowndll))
+
     @unittest.skipUnless(libc_name is not None, 'could not find libc')
     @unittest.skipUnless(libc_name is not None and
                          os.path.basename(libc_name) == "libc.so.6",
diff --git a/lib-python/2.7/ctypes/test/test_parameters.py b/lib-python/2.7/ctypes/test/test_parameters.py
--- a/lib-python/2.7/ctypes/test/test_parameters.py
+++ b/lib-python/2.7/ctypes/test/test_parameters.py
@@ -1,5 +1,6 @@
 import unittest, sys
 from ctypes.test import need_symbol
+import test.support
 
 class SimpleTypesTestCase(unittest.TestCase):
 
@@ -174,6 +175,36 @@
         # ArgumentError: argument 1: ValueError: 99
         self.assertRaises(ArgumentError, func, 99)
 
+    def test_abstract(self):
+        from ctypes import (Array, Structure, Union, _Pointer,
+                            _SimpleCData, _CFuncPtr)
+
+        self.assertRaises(TypeError, Array.from_param, 42)
+        self.assertRaises(TypeError, Structure.from_param, 42)
+        self.assertRaises(TypeError, Union.from_param, 42)
+        self.assertRaises(TypeError, _CFuncPtr.from_param, 42)
+        self.assertRaises(TypeError, _Pointer.from_param, 42)
+        self.assertRaises(TypeError, _SimpleCData.from_param, 42)
+
+    @test.support.cpython_only
+    def test_issue31311(self):
+        # __setstate__ should neither raise a SystemError nor crash in case
+        # of a bad __dict__.
+        from ctypes import Structure
+
+        class BadStruct(Structure):
+            @property
+            def __dict__(self):
+                pass
+        with self.assertRaises(TypeError):
+            BadStruct().__setstate__({}, b'foo')
+
+        class WorseStruct(Structure):
+            @property
+            def __dict__(self):
+                1/0.0
+        with self.assertRaises(ZeroDivisionError):
+            WorseStruct().__setstate__({}, b'foo')
 
 ################################################################
 
diff --git a/lib-python/2.7/ctypes/test/test_pep3118.py b/lib-python/2.7/ctypes/test/test_pep3118.py
--- a/lib-python/2.7/ctypes/test/test_pep3118.py
+++ b/lib-python/2.7/ctypes/test/test_pep3118.py
@@ -109,6 +109,34 @@
 # This table contains format strings as they look on little endian
 # machines.  The test replaces '<' with '>' on big endian machines.
 #
+
+# Platform-specific type codes
+s_bool = {1: '?', 2: 'H', 4: 'L', 8: 'Q'}[sizeof(c_bool)]
+s_short = {2: 'h', 4: 'l', 8: 'q'}[sizeof(c_short)]
+s_ushort = {2: 'H', 4: 'L', 8: 'Q'}[sizeof(c_ushort)]
+s_int = {2: 'h', 4: 'i', 8: 'q'}[sizeof(c_int)]
+s_uint = {2: 'H', 4: 'I', 8: 'Q'}[sizeof(c_uint)]
+s_long = {4: 'l', 8: 'q'}[sizeof(c_long)]
+s_ulong = {4: 'L', 8: 'Q'}[sizeof(c_ulong)]
+s_longlong = "q"
+s_ulonglong = "Q"
+s_float = "f"
+s_double = "d"
+s_longdouble = "g"
+
+# Alias definitions in ctypes/__init__.py
+if c_int is c_long:
+    s_int = s_long
+if c_uint is c_ulong:
+    s_uint = s_ulong
+if c_longlong is c_long:
+    s_longlong = s_long
+if c_ulonglong is c_ulong:
+    s_ulonglong = s_ulong
+if c_longdouble is c_double:
+    s_longdouble = s_double
+
+
 native_types = [
     # type                      format                  shape           calc itemsize
 
@@ -117,52 +145,51 @@
     (c_char,                    "<c",                   None,           c_char),
     (c_byte,                    "<b",                   None,           c_byte),
     (c_ubyte,                   "<B",                   None,           c_ubyte),
-    (c_short,                   "<h",                   None,           c_short),
-    (c_ushort,                  "<H",                   None,           c_ushort),
+    (c_short,                   "<" + s_short,          None,           c_short),
+    (c_ushort,                  "<" + s_ushort,         None,           c_ushort),
 
-    # c_int and c_uint may be aliases to c_long
-    #(c_int,                     "<i",                   None,           c_int),
-    #(c_uint,                    "<I",                   None,           c_uint),
+    (c_int,                     "<" + s_int,            None,           c_int),
+    (c_uint,                    "<" + s_uint,           None,           c_uint),
 
-    (c_long,                    "<l",                   None,           c_long),
-    (c_ulong,                   "<L",                   None,           c_ulong),
+    (c_long,                    "<" + s_long,           None,           c_long),
+    (c_ulong,                   "<" + s_ulong,          None,           c_ulong),
 
-    # c_longlong and c_ulonglong are aliases on 64-bit platforms
-    #(c_longlong,                "<q",                   None,           c_longlong),
-    #(c_ulonglong,               "<Q",                   None,           c_ulonglong),
+    (c_longlong,                "<" + s_longlong,       None,           c_longlong),
+    (c_ulonglong,               "<" + s_ulonglong,      None,           c_ulonglong),
 
     (c_float,                   "<f",                   None,           c_float),
     (c_double,                  "<d",                   None,           c_double),
-    # c_longdouble may be an alias to c_double
 
-    (c_bool,                    "<?",                   None,           c_bool),
+    (c_longdouble,              "<" + s_longdouble,     None,           c_longdouble),
+
+    (c_bool,                    "<" + s_bool,           None,           c_bool),
     (py_object,                 "<O",                   None,           py_object),
 
     ## pointers
 
     (POINTER(c_byte),           "&<b",                  None,           POINTER(c_byte)),
-    (POINTER(POINTER(c_long)),  "&&<l",                 None,           POINTER(POINTER(c_long))),
+    (POINTER(POINTER(c_long)),  "&&<" + s_long,         None,           POINTER(POINTER(c_long))),
 
     ## arrays and pointers
 
     (c_double * 4,              "<d",                   (4,),           c_double),
     (c_float * 4 * 3 * 2,       "<f",                   (2,3,4),        c_float),
-    (POINTER(c_short) * 2,      "&<h",                  (2,),           POINTER(c_short)),
-    (POINTER(c_short) * 2 * 3,  "&<h",                  (3,2,),         POINTER(c_short)),
-    (POINTER(c_short * 2),      "&(2)<h",               None,           POINTER(c_short)),
+    (POINTER(c_short) * 2,      "&<" + s_short,         (2,),           POINTER(c_short)),
+    (POINTER(c_short) * 2 * 3,  "&<" + s_short,         (3,2,),         POINTER(c_short)),
+    (POINTER(c_short * 2),      "&(2)<" + s_short,      None,             POINTER(c_short)),
 
     ## structures and unions
 
-    (Point,                     "T{<l:x:<l:y:}",        None,           Point),
+    (Point,                     "T{<l:x:<l:y:}".replace('l', s_long),  None,  Point),
     # packed structures do not implement the pep
-    (PackedPoint,               "B",                    None,           PackedPoint),
-    (Point2,                    "T{<l:x:<l:y:}",        None,           Point2),
-    (EmptyStruct,               "T{}",                  None,           EmptyStruct),
+    (PackedPoint,               "B",                                   None,  PackedPoint),
+    (Point2,                    "T{<l:x:<l:y:}".replace('l', s_long),  None,  Point2),
+    (EmptyStruct,               "T{}",                                 None,  EmptyStruct),
     # the pep does't support unions
-    (aUnion,                    "B",                    None,           aUnion),
+    (aUnion,                    "B",                                   None,  aUnion),
     # structure with sub-arrays
-    (StructWithArrays,          "T{(2,3)<l:x:(4)T{<l:x:<l:y:}:y:}", None,  StructWithArrays),
-    (StructWithArrays * 3,      "T{(2,3)<l:x:(4)T{<l:x:<l:y:}:y:}", (3,),  StructWithArrays),
+    (StructWithArrays, "T{(2,3)<l:x:(4)T{<l:x:<l:y:}:y:}".replace('l', s_long), None, StructWithArrays),
+    (StructWithArrays * 3, "T{(2,3)<l:x:(4)T{<l:x:<l:y:}:y:}".replace('l', s_long), (3,), StructWithArrays),
 
     ## pointer to incomplete structure
     (Incomplete,                "B",                    None,           Incomplete),
@@ -170,7 +197,7 @@
 
     # 'Complete' is a structure that starts incomplete, but is completed after the
     # pointer type to it has been created.
-    (Complete,                  "T{<l:a:}",             None,           Complete),
+    (Complete,                  "T{<l:a:}".replace('l', s_long), None, Complete),
     # Unfortunately the pointer format string is not fixed...
     (POINTER(Complete),         "&B",                   None,           POINTER(Complete)),
 
@@ -193,10 +220,10 @@
 # and little endian machines.
 #
 endian_types = [
-    (BEPoint,                   "T{>l:x:>l:y:}",        None,           BEPoint),
-    (LEPoint,                   "T{<l:x:<l:y:}",        None,           LEPoint),
-    (POINTER(BEPoint),          "&T{>l:x:>l:y:}",       None,           POINTER(BEPoint)),
-    (POINTER(LEPoint),          "&T{<l:x:<l:y:}",       None,           POINTER(LEPoint)),
+    (BEPoint, "T{>l:x:>l:y:}".replace('l', s_long), None, BEPoint),
+    (LEPoint, "T{<l:x:<l:y:}".replace('l', s_long), None, LEPoint),
+    (POINTER(BEPoint), "&T{>l:x:>l:y:}".replace('l', s_long), None, POINTER(BEPoint)),
+    (POINTER(LEPoint), "&T{<l:x:<l:y:}".replace('l', s_long), None, POINTER(LEPoint)),
     ]
 
 if __name__ == "__main__":
diff --git a/lib-python/2.7/ctypes/test/test_pointers.py b/lib-python/2.7/ctypes/test/test_pointers.py
--- a/lib-python/2.7/ctypes/test/test_pointers.py
+++ b/lib-python/2.7/ctypes/test/test_pointers.py
@@ -210,6 +210,11 @@
         from ctypes import _pointer_type_cache
         del _pointer_type_cache[id(P)]
 
+    def test_abstract(self):
+        from ctypes import _Pointer
+
+        self.assertRaises(TypeError, _Pointer.set_type, 42)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_strings.py b/lib-python/2.7/ctypes/test/test_strings.py
--- a/lib-python/2.7/ctypes/test/test_strings.py
+++ b/lib-python/2.7/ctypes/test/test_strings.py
@@ -61,6 +61,13 @@
 ##        print BUF.from_param(c_char_p("python"))
 ##        print BUF.from_param(BUF(*"pyth"))
 
+    def test_del_segfault(self):
+        BUF = c_char * 4
+        buf = BUF()
+        with self.assertRaises(AttributeError):
+            del buf.raw
+
+
 @need_symbol('c_wchar')
 class WStringArrayTestCase(unittest.TestCase):
     def test(self):
diff --git a/lib-python/2.7/ctypes/test/test_struct_fields.py b/lib-python/2.7/ctypes/test/test_struct_fields.py
--- a/lib-python/2.7/ctypes/test/test_struct_fields.py
+++ b/lib-python/2.7/ctypes/test/test_struct_fields.py
@@ -46,5 +46,29 @@
         Y._fields_ = []
         self.assertRaises(AttributeError, setattr, X, "_fields_", [])
 
+    # __set__ and __get__ should raise a TypeError in case their self
+    # argument is not a ctype instance.
+    def test___set__(self):
+        class MyCStruct(Structure):
+            _fields_ = (("field", c_int),)
+        self.assertRaises(TypeError,
+                          MyCStruct.field.__set__, 'wrong type self', 42)
+
+        class MyCUnion(Union):
+            _fields_ = (("field", c_int),)
+        self.assertRaises(TypeError,
+                          MyCUnion.field.__set__, 'wrong type self', 42)
+
+    def test___get__(self):
+        class MyCStruct(Structure):
+            _fields_ = (("field", c_int),)
+        self.assertRaises(TypeError,
+                          MyCStruct.field.__get__, 'wrong type self', 42)
+
+        class MyCUnion(Union):
+            _fields_ = (("field", c_int),)
+        self.assertRaises(TypeError,
+                          MyCUnion.field.__get__, 'wrong type self', 42)
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_structures.py b/lib-python/2.7/ctypes/test/test_structures.py
--- a/lib-python/2.7/ctypes/test/test_structures.py
+++ b/lib-python/2.7/ctypes/test/test_structures.py
@@ -3,6 +3,7 @@
 from ctypes.test import need_symbol
 from struct import calcsize
 import _testcapi
+import _ctypes_test
 
 class SubclassesTest(unittest.TestCase):
     def test_subclass(self):
@@ -401,6 +402,28 @@
                          (1, 0, 0, 0, 0, 0))
         self.assertRaises(TypeError, lambda: Z(1, 2, 3, 4, 5, 6, 7))
 
+    def test_pass_by_value(self):
+        # This should mirror the structure in Modules/_ctypes/_ctypes_test.c
+        class X(Structure):
+            _fields_ = [
+                ('first', c_ulong),
+                ('second', c_ulong),
+                ('third', c_ulong),
+            ]
+
+        s = X()
+        s.first = 0xdeadbeef
+        s.second = 0xcafebabe
+        s.third = 0x0bad1dea
+        dll = CDLL(_ctypes_test.__file__)
+        func = dll._testfunc_large_struct_update_value
+        func.argtypes = (X,)
+        func.restype = None
+        func(s)
+        self.assertEqual(s.first, 0xdeadbeef)
+        self.assertEqual(s.second, 0xcafebabe)
+        self.assertEqual(s.third, 0x0bad1dea)
+
 class PointerMemberTestCase(unittest.TestCase):
 
     def test(self):
diff --git a/lib-python/2.7/ctypes/test/test_win32.py b/lib-python/2.7/ctypes/test/test_win32.py
--- a/lib-python/2.7/ctypes/test/test_win32.py
+++ b/lib-python/2.7/ctypes/test/test_win32.py
@@ -53,6 +53,24 @@
         windll.user32.GetDesktopWindow()
 
 @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test')
+class ReturnStructSizesTestCase(unittest.TestCase):
+    def test_sizes(self):
+        dll = CDLL(_ctypes_test.__file__)
+        for i in range(1, 11):
+            fields = [ ("f%d" % f, c_char) for f in range(1, i + 1)]
+            class S(Structure):
+                _fields_ = fields
+            f = getattr(dll, "TestSize%d" % i)
+            f.restype = S
+            res = f()
+            for i, f in enumerate(fields):
+                value = getattr(res, f[0])
+                expected = chr(ord('a') + i)
+                self.assertEquals(value, expected)
+
+
+
+ at unittest.skipUnless(sys.platform == "win32", 'Windows-specific test')
 class TestWintypes(unittest.TestCase):
     def test_HWND(self):
         from ctypes import wintypes
diff --git a/lib-python/2.7/curses/ascii.py b/lib-python/2.7/curses/ascii.py
--- a/lib-python/2.7/curses/ascii.py
+++ b/lib-python/2.7/curses/ascii.py
@@ -53,19 +53,19 @@
 
 def isalnum(c): return isalpha(c) or isdigit(c)
 def isalpha(c): return isupper(c) or islower(c)
-def isascii(c): return _ctoi(c) <= 127          # ?
+def isascii(c): return 0 <= _ctoi(c) <= 127          # ?
 def isblank(c): return _ctoi(c) in (9, 32)
-def iscntrl(c): return _ctoi(c) <= 31 or _ctoi(c) == 127
-def isdigit(c): return _ctoi(c) >= 48 and _ctoi(c) <= 57
-def isgraph(c): return _ctoi(c) >= 33 and _ctoi(c) <= 126
-def islower(c): return _ctoi(c) >= 97 and _ctoi(c) <= 122
-def isprint(c): return _ctoi(c) >= 32 and _ctoi(c) <= 126
+def iscntrl(c): return 0 <= _ctoi(c) <= 31 or _ctoi(c) == 127
+def isdigit(c): return 48 <= _ctoi(c) <= 57
+def isgraph(c): return 33 <= _ctoi(c) <= 126
+def islower(c): return 97 <= _ctoi(c) <= 122
+def isprint(c): return 32 <= _ctoi(c) <= 126
 def ispunct(c): return isgraph(c) and not isalnum(c)
 def isspace(c): return _ctoi(c) in (9, 10, 11, 12, 13, 32)
-def isupper(c): return _ctoi(c) >= 65 and _ctoi(c) <= 90
+def isupper(c): return 65 <= _ctoi(c) <= 90
 def isxdigit(c): return isdigit(c) or \
-    (_ctoi(c) >= 65 and _ctoi(c) <= 70) or (_ctoi(c) >= 97 and _ctoi(c) <= 102)
-def isctrl(c): return _ctoi(c) < 32
+    (65 <= _ctoi(c) <= 70) or (97 <= _ctoi(c) <= 102)
+def isctrl(c): return 0 <= _ctoi(c) < 32
 def ismeta(c): return _ctoi(c) > 127
 
 def ascii(c):
diff --git a/lib-python/2.7/curses/has_key.py b/lib-python/2.7/curses/has_key.py
--- a/lib-python/2.7/curses/has_key.py
+++ b/lib-python/2.7/curses/has_key.py
@@ -182,7 +182,7 @@
         L = []
         _curses.initscr()
         for key in _capability_names.keys():
-            system = key in _curses
+            system = _curses.has_key(key)
             python = has_key(key)
             if system != python:
                 L.append( 'Mismatch for key %s, system=%i, Python=%i'
diff --git a/lib-python/2.7/curses/textpad.py b/lib-python/2.7/curses/textpad.py
--- a/lib-python/2.7/curses/textpad.py
+++ b/lib-python/2.7/curses/textpad.py
@@ -43,16 +43,20 @@
     def __init__(self, win, insert_mode=False):
         self.win = win
         self.insert_mode = insert_mode
-        (self.maxy, self.maxx) = win.getmaxyx()
-        self.maxy = self.maxy - 1
-        self.maxx = self.maxx - 1
+        self._update_max_yx()
         self.stripspaces = 1
         self.lastcmd = None
         win.keypad(1)
 
+    def _update_max_yx(self):
+        maxy, maxx = self.win.getmaxyx()
+        self.maxy = maxy - 1
+        self.maxx = maxx - 1
+
     def _end_of_line(self, y):
         """Go to the location of the first blank on the given line,
         returning the index of the last non-blank character."""
+        self._update_max_yx()
         last = self.maxx
         while True:
             if curses.ascii.ascii(self.win.inch(y, last)) != curses.ascii.SP:
@@ -64,8 +68,10 @@
         return last
 
     def _insert_printable_char(self, ch):
+        self._update_max_yx()
         (y, x) = self.win.getyx()
-        if y < self.maxy or x < self.maxx:
+        backyx = None
+        while y < self.maxy or x < self.maxx:
             if self.insert_mode:
                 oldch = self.win.inch()
             # The try-catch ignores the error we trigger from some curses
@@ -75,14 +81,20 @@
                 self.win.addch(ch)
             except curses.error:
                 pass
-            if self.insert_mode:
-                (backy, backx) = self.win.getyx()
-                if curses.ascii.isprint(oldch):
-                    self._insert_printable_char(oldch)
-                    self.win.move(backy, backx)
+            if not self.insert_mode or not curses.ascii.isprint(oldch):
+                break
+            ch = oldch
+            (y, x) = self.win.getyx()
+            # Remember where to put the cursor back since we are in insert_mode
+            if backyx is None:
+                backyx = y, x
+
+        if backyx is not None:
+            self.win.move(*backyx)
 
     def do_command(self, ch):
         "Process a single editing command."
+        self._update_max_yx()
         (y, x) = self.win.getyx()
         self.lastcmd = ch
         if curses.ascii.isprint(ch):
@@ -148,6 +160,7 @@
     def gather(self):
         "Collect and return the contents of the window."
         result = ""
+        self._update_max_yx()
         for y in range(self.maxy+1):
             self.win.move(y, 0)
             stop = self._end_of_line(y)
diff --git a/lib-python/2.7/decimal.py b/lib-python/2.7/decimal.py
--- a/lib-python/2.7/decimal.py
+++ b/lib-python/2.7/decimal.py
@@ -1909,7 +1909,7 @@
         if not other and not self:
             return context._raise_error(InvalidOperation,
                                         'at least one of pow() 1st argument '
-                                        'and 2nd argument must be nonzero ;'
+                                        'and 2nd argument must be nonzero; '
                                         '0**0 is not defined')
 
         # compute sign of result
diff --git a/lib-python/2.7/difflib.py b/lib-python/2.7/difflib.py
--- a/lib-python/2.7/difflib.py
+++ b/lib-python/2.7/difflib.py
@@ -1103,7 +1103,7 @@
 
 import re
 
-def IS_LINE_JUNK(line, pat=re.compile(r"\s*#?\s*$").match):
+def IS_LINE_JUNK(line, pat=re.compile(r"\s*(?:#\s*)?$").match):
     r"""
     Return 1 for ignorable line: iff `line` is blank or contains a single '#'.
 
diff --git a/lib-python/2.7/distutils/archive_util.py b/lib-python/2.7/distutils/archive_util.py
--- a/lib-python/2.7/distutils/archive_util.py
+++ b/lib-python/2.7/distutils/archive_util.py
@@ -162,7 +162,15 @@
             zip = zipfile.ZipFile(zip_filename, "w",
                                   compression=zipfile.ZIP_DEFLATED)
 
+            if base_dir != os.curdir:
+                path = os.path.normpath(os.path.join(base_dir, ''))
+                zip.write(path, path)
+                log.info("adding '%s'", path)
             for dirpath, dirnames, filenames in os.walk(base_dir):
+                for name in dirnames:
+                    path = os.path.normpath(os.path.join(dirpath, name, ''))
+                    zip.write(path, path)
+                    log.info("adding '%s'", path)
                 for name in filenames:
                     path = os.path.normpath(os.path.join(dirpath, name))
                     if os.path.isfile(path):
diff --git a/lib-python/2.7/distutils/ccompiler.py b/lib-python/2.7/distutils/ccompiler.py
--- a/lib-python/2.7/distutils/ccompiler.py
+++ b/lib-python/2.7/distutils/ccompiler.py
@@ -160,7 +160,7 @@
             self.set_executable(key, args[key])
 
     def set_executable(self, key, value):
-        if isinstance(value, str):
+        if isinstance(value, basestring):
             setattr(self, key, split_quoted(value))
         else:
             setattr(self, key, value)
diff --git a/lib-python/2.7/distutils/command/bdist_dumb.py b/lib-python/2.7/distutils/command/bdist_dumb.py
--- a/lib-python/2.7/distutils/command/bdist_dumb.py
+++ b/lib-python/2.7/distutils/command/bdist_dumb.py
@@ -35,7 +35,7 @@
                     ('skip-build', None,
                      "skip rebuilding everything (for testing/debugging)"),
                     ('relative', None,
-                     "build the archive using relative paths"
+                     "build the archive using relative paths "
                      "(default: false)"),
                     ('owner=', 'u',
                      "Owner name used when creating a tar file"
diff --git a/lib-python/2.7/distutils/command/bdist_msi.py b/lib-python/2.7/distutils/command/bdist_msi.py
--- a/lib-python/2.7/distutils/command/bdist_msi.py
+++ b/lib-python/2.7/distutils/command/bdist_msi.py
@@ -99,14 +99,14 @@
                     ('no-target-compile', 'c',
                      "do not compile .py to .pyc on the target system"),
                     ('no-target-optimize', 'o',
-                     "do not compile .py to .pyo (optimized)"
+                     "do not compile .py to .pyo (optimized) "
                      "on the target system"),
                     ('dist-dir=', 'd',
                      "directory to put final built distributions in"),
                     ('skip-build', None,
                      "skip rebuilding everything (for testing/debugging)"),
                     ('install-script=', None,
-                     "basename of installation script to be run after"
+                     "basename of installation script to be run after "
                      "installation or before deinstallation"),
                     ('pre-install-script=', None,
                      "Fully qualified filename of a script to be run before "
diff --git a/lib-python/2.7/distutils/command/bdist_rpm.py b/lib-python/2.7/distutils/command/bdist_rpm.py
--- a/lib-python/2.7/distutils/command/bdist_rpm.py
+++ b/lib-python/2.7/distutils/command/bdist_rpm.py
@@ -63,7 +63,7 @@
          "RPM \"vendor\" (eg. \"Joe Blow <joe at example.com>\") "
          "[default: maintainer or author from setup script]"),
         ('packager=', None,
-         "RPM packager (eg. \"Jane Doe <jane at example.net>\")"
+         "RPM packager (eg. \"Jane Doe <jane at example.net>\") "
          "[default: vendor]"),
         ('doc-files=', None,
          "list of documentation files (space or comma-separated)"),
diff --git a/lib-python/2.7/distutils/command/bdist_wininst.py b/lib-python/2.7/distutils/command/bdist_wininst.py
--- a/lib-python/2.7/distutils/command/bdist_wininst.py
+++ b/lib-python/2.7/distutils/command/bdist_wininst.py
@@ -35,7 +35,7 @@
                     ('no-target-compile', 'c',
                      "do not compile .py to .pyc on the target system"),
                     ('no-target-optimize', 'o',
-                     "do not compile .py to .pyo (optimized)"
+                     "do not compile .py to .pyo (optimized) "
                      "on the target system"),
                     ('dist-dir=', 'd',
                      "directory to put final built distributions in"),
@@ -46,7 +46,7 @@
                     ('skip-build', None,
                      "skip rebuilding everything (for testing/debugging)"),
                     ('install-script=', None,
-                     "basename of installation script to be run after"
+                     "basename of installation script to be run after "
                      "installation or before deinstallation"),
                     ('pre-install-script=', None,
                      "Fully qualified filename of a script to be run before "
diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py
--- a/lib-python/2.7/distutils/command/build_ext.py
+++ b/lib-python/2.7/distutils/command/build_ext.py
@@ -366,7 +366,7 @@
             ext_name, build_info = ext
 
             log.warn(("old-style (ext_name, build_info) tuple found in "
-                      "ext_modules for extension '%s'"
+                      "ext_modules for extension '%s' "
                       "-- please convert to Extension instance" % ext_name))
 
             if not (isinstance(ext_name, str) and
diff --git a/lib-python/2.7/distutils/command/upload.py b/lib-python/2.7/distutils/command/upload.py
--- a/lib-python/2.7/distutils/command/upload.py
+++ b/lib-python/2.7/distutils/command/upload.py
@@ -55,7 +55,9 @@
 
     def run(self):
         if not self.distribution.dist_files:
-            raise DistutilsOptionError("No dist file created in earlier command")
+            msg = ("Must create and upload files in one command "
+                   "(e.g. setup.py sdist upload)")
+            raise DistutilsOptionError(msg)
         for command, pyversion, filename in self.distribution.dist_files:
             self.upload_file(command, pyversion, filename)
 
@@ -155,8 +157,6 @@
                 body.write(fn)
                 body.write("\r\n\r\n")
                 body.write(value)
-                if value and value[-1] == '\r':
-                    body.write('\n')  # write an extra newline (lurve Macs)
         body.write(end_boundary)
         body = body.getvalue()
 
diff --git a/lib-python/2.7/distutils/spawn.py b/lib-python/2.7/distutils/spawn.py
--- a/lib-python/2.7/distutils/spawn.py
+++ b/lib-python/2.7/distutils/spawn.py
@@ -208,7 +208,8 @@
     os.environ['PATH'].  Returns the complete filename or None if not found.
     """
     if path is None:
-        path = os.environ['PATH']
+        path = os.environ.get('PATH', os.defpath)
+
     paths = path.split(os.pathsep)
     base, ext = os.path.splitext(executable)
 
diff --git a/lib-python/2.7/distutils/tests/test_archive_util.py b/lib-python/2.7/distutils/tests/test_archive_util.py
--- a/lib-python/2.7/distutils/tests/test_archive_util.py
+++ b/lib-python/2.7/distutils/tests/test_archive_util.py
@@ -98,7 +98,7 @@
         try:
             names = tar.getnames()
             names.sort()
-            return tuple(names)
+            return names
         finally:
             tar.close()
 
diff --git a/lib-python/2.7/distutils/tests/test_bdist_dumb.py b/lib-python/2.7/distutils/tests/test_bdist_dumb.py
--- a/lib-python/2.7/distutils/tests/test_bdist_dumb.py
+++ b/lib-python/2.7/distutils/tests/test_bdist_dumb.py
@@ -86,7 +86,7 @@
         finally:
             fp.close()
 
-        contents = sorted(os.path.basename(fn) for fn in contents)
+        contents = sorted(filter(None, map(os.path.basename, contents)))
         wanted = ['foo-0.1-py%s.%s.egg-info' % sys.version_info[:2], 'foo.py']
         if not sys.dont_write_bytecode:
             wanted.append('foo.pyc')
diff --git a/lib-python/2.7/distutils/tests/test_bdist_rpm.py b/lib-python/2.7/distutils/tests/test_bdist_rpm.py
--- a/lib-python/2.7/distutils/tests/test_bdist_rpm.py
+++ b/lib-python/2.7/distutils/tests/test_bdist_rpm.py
@@ -99,7 +99,7 @@
     @unittest.skipIf(find_executable('rpmbuild') is None,
                      'the rpmbuild command is not found')
     def test_no_optimize_flag(self):
-        # let's create a package that brakes bdist_rpm
+        # let's create a package that breaks bdist_rpm
         tmp_dir = self.mkdtemp()
         os.environ['HOME'] = tmp_dir   # to confine dir '.rpmdb' creation
         pkg_dir = os.path.join(tmp_dir, 'foo')
diff --git a/lib-python/2.7/distutils/tests/test_build_ext.py b/lib-python/2.7/distutils/tests/test_build_ext.py
--- a/lib-python/2.7/distutils/tests/test_build_ext.py
+++ b/lib-python/2.7/distutils/tests/test_build_ext.py
@@ -20,6 +20,7 @@
 
 class BuildExtTestCase(support.TempdirManager,
                        support.LoggingSilencer,
+                       support.EnvironGuard,
                        unittest.TestCase):
     def setUp(self):
         super(BuildExtTestCase, self).setUp()
diff --git a/lib-python/2.7/distutils/tests/test_ccompiler.py b/lib-python/2.7/distutils/tests/test_ccompiler.py
--- a/lib-python/2.7/distutils/tests/test_ccompiler.py
+++ b/lib-python/2.7/distutils/tests/test_ccompiler.py
@@ -24,6 +24,30 @@
 
 class CCompilerTestCase(support.EnvironGuard, unittest.TestCase):
 
+    def test_set_executables(self):
+        class MyCCompiler(CCompiler):
+            executables = {'compiler': '', 'compiler_cxx': '', 'linker': ''}
+
+        compiler = MyCCompiler()
+
+        # set executable as list
+        compiler.set_executables(compiler=['env', 'OMPI_MPICC=clang', 'mpicc'])
+        self.assertEqual(compiler.compiler, ['env',
+                                             'OMPI_MPICC=clang',
+                                             'mpicc'])
+
+        # set executable as string
+        compiler.set_executables(compiler_cxx='env OMPI_MPICXX=clang++ mpicxx')
+        self.assertEqual(compiler.compiler_cxx, ['env',
+                                                 'OMPI_MPICXX=clang++',
+                                                 'mpicxx'])
+
+        # set executable as unicode string
+        compiler.set_executables(linker=u'env OMPI_MPICXX=clang++ mpiCC')
+        self.assertEqual(compiler.linker, [u'env',
+                                           u'OMPI_MPICXX=clang++',
+                                           u'mpiCC'])
+
     def test_gen_lib_options(self):
         compiler = FakeCompiler()
         libdirs = ['lib1', 'lib2']
diff --git a/lib-python/2.7/distutils/tests/test_check.py b/lib-python/2.7/distutils/tests/test_check.py
--- a/lib-python/2.7/distutils/tests/test_check.py
+++ b/lib-python/2.7/distutils/tests/test_check.py
@@ -8,6 +8,12 @@
 from distutils.tests import support
 from distutils.errors import DistutilsSetupError
 
+try:
+    import pygments
+except ImportError:
+    pygments = None
+
+
 class CheckTestCase(support.LoggingSilencer,
                     support.TempdirManager,
                     unittest.TestCase):
@@ -120,9 +126,15 @@
             pkg_info, dist = self.create_dist(long_description=rest_with_code)
             cmd = check(dist)
             cmd.check_restructuredtext()
-            self.assertEqual(cmd._warnings, 0)
             msgs = cmd._check_rst_data(rest_with_code)
-            self.assertEqual(len(msgs), 0)
+            if pygments is not None:
+                self.assertEqual(len(msgs), 0)
+            else:
+                self.assertEqual(len(msgs), 1)
+                self.assertEqual(
+                    str(msgs[0][1]),
+                    'Cannot analyze code. Pygments package not found.'
+                )
 
     def test_check_all(self):
 
diff --git a/lib-python/2.7/distutils/tests/test_install.py b/lib-python/2.7/distutils/tests/test_install.py
--- a/lib-python/2.7/distutils/tests/test_install.py
+++ b/lib-python/2.7/distutils/tests/test_install.py
@@ -26,6 +26,7 @@
 
 
 class InstallTestCase(support.TempdirManager,
+                      support.EnvironGuard,
                       support.LoggingSilencer,
                       unittest.TestCase):
 
diff --git a/lib-python/2.7/distutils/tests/test_sdist.py b/lib-python/2.7/distutils/tests/test_sdist.py
--- a/lib-python/2.7/distutils/tests/test_sdist.py
+++ b/lib-python/2.7/distutils/tests/test_sdist.py
@@ -130,7 +130,9 @@
             zip_file.close()
 
         # making sure everything has been pruned correctly
-        self.assertEqual(len(content), 4)
+        expected = ['', 'PKG-INFO', 'README', 'setup.py',
+                    'somecode/', 'somecode/__init__.py']
+        self.assertEqual(sorted(content), ['fake-1.0/' + x for x in expected])
 
     @unittest.skipUnless(zlib, "requires zlib")
     def test_make_distribution(self):
@@ -246,7 +248,13 @@
             zip_file.close()
 
         # making sure everything was added
-        self.assertEqual(len(content), 12)
+        expected = ['', 'PKG-INFO', 'README', 'buildout.cfg',
+                    'data/', 'data/data.dt', 'inroot.txt',
+                    'scripts/', 'scripts/script.py', 'setup.py',
+                    'some/', 'some/file.txt', 'some/other_file.txt',
+                    'somecode/', 'somecode/__init__.py', 'somecode/doc.dat',
+                    'somecode/doc.txt']
+        self.assertEqual(sorted(content), ['fake-1.0/' + x for x in expected])
 
         # checking the MANIFEST
         f = open(join(self.tmp_dir, 'MANIFEST'))
diff --git a/lib-python/2.7/distutils/tests/test_spawn.py b/lib-python/2.7/distutils/tests/test_spawn.py
--- a/lib-python/2.7/distutils/tests/test_spawn.py
+++ b/lib-python/2.7/distutils/tests/test_spawn.py
@@ -1,8 +1,11 @@
 """Tests for distutils.spawn."""
+import os
+import stat
+import sys
+import time
 import unittest
-import os
-import time
-from test.test_support import captured_stdout, run_unittest
+from test.support import captured_stdout, run_unittest
+from test import support as test_support
 
 from distutils.spawn import _nt_quote_args
 from distutils.spawn import spawn, find_executable
@@ -53,6 +56,48 @@
         os.chmod(exe, 0777)
         spawn([exe])  # should work without any error
 
+    def test_find_executable(self):
+        with test_support.temp_dir() as tmp_dir:
+            # use TESTFN to get a pseudo-unique filename
+            program_noeext = test_support.TESTFN
+            # Give the temporary program an ".exe" suffix for all.
+            # It's needed on Windows and not harmful on other platforms.
+            program = program_noeext + ".exe"
+
+            filename = os.path.join(tmp_dir, program)
+            with open(filename, "wb"):
+                pass
+            os.chmod(filename, stat.S_IXUSR)
+
+            # test path parameter
+            rv = find_executable(program, path=tmp_dir)
+            self.assertEqual(rv, filename)
+
+            if sys.platform == 'win32':
+                # test without ".exe" extension
+                rv = find_executable(program_noeext, path=tmp_dir)
+                self.assertEqual(rv, filename)
+
+            # test find in the current directory
+            with test_support.change_cwd(tmp_dir):
+                rv = find_executable(program)
+                self.assertEqual(rv, program)
+
+            # test non-existent program
+            dont_exist_program = "dontexist_" + program
+            rv = find_executable(dont_exist_program , path=tmp_dir)
+            self.assertIsNone(rv)
+
+            # test os.defpath: missing PATH environment variable
+            with test_support.EnvironmentVarGuard() as env:
+                from distutils import spawn
+                with test_support.swap_attr(spawn.os, 'defpath', tmp_dir):
+                    env.pop('PATH')
+
+                    rv = find_executable(program)
+                    self.assertEqual(rv, filename)
+
+
 def test_suite():
     return unittest.makeSuite(SpawnTestCase)
 
diff --git a/lib-python/2.7/distutils/tests/test_upload.py b/lib-python/2.7/distutils/tests/test_upload.py
--- a/lib-python/2.7/distutils/tests/test_upload.py
+++ b/lib-python/2.7/distutils/tests/test_upload.py
@@ -128,6 +128,32 @@
         auth = self.last_open.req.headers['Authorization']
         self.assertNotIn('\n', auth)
 
+    # bpo-32304: archives whose last byte was b'\r' were corrupted due to
+    # normalization intended for Mac OS 9.
+    def test_upload_correct_cr(self):
+        # content that ends with \r should not be modified.
+        tmp = self.mkdtemp()
+        path = os.path.join(tmp, 'xxx')
+        self.write_file(path, content='yy\r')

From pypy.commits at gmail.com  Mon May 20 07:30:42 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 20 May 2019 04:30:42 -0700 (PDT)
Subject: [pypy-commit] pypy stdlib-2.7.16: merge new stdlib into branch
Message-ID: <5ce28fe2.1c69fb81.fcf0b.0edc@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: stdlib-2.7.16
Changeset: r96643:c8bdc9146f6e
Date: 2019-05-20 14:05 +0300
http://bitbucket.org/pypy/pypy/changeset/c8bdc9146f6e/

Log:	merge new stdlib into branch

diff too long, truncating to 2000 out of 46648 lines

diff --git a/lib-python/2.7/HTMLParser.py b/lib-python/2.7/HTMLParser.py
--- a/lib-python/2.7/HTMLParser.py
+++ b/lib-python/2.7/HTMLParser.py
@@ -462,11 +462,12 @@
             else:
                 # Cannot use name2codepoint directly, because HTMLParser supports apos,
                 # which is not part of HTML 4
-                import htmlentitydefs
                 if HTMLParser.entitydefs is None:
-                    entitydefs = HTMLParser.entitydefs = {'apos':u"'"}
+                    import htmlentitydefs
+                    entitydefs = {'apos':u"'"}
                     for k, v in htmlentitydefs.name2codepoint.iteritems():
                         entitydefs[k] = unichr(v)
+                    HTMLParser.entitydefs = entitydefs
                 try:
                     return self.entitydefs[s]
                 except KeyError:
diff --git a/lib-python/2.7/SocketServer.py b/lib-python/2.7/SocketServer.py
--- a/lib-python/2.7/SocketServer.py
+++ b/lib-python/2.7/SocketServer.py
@@ -229,6 +229,9 @@
                 # shutdown request and wastes cpu at all other times.
                 r, w, e = _eintr_retry(select.select, [self], [], [],
                                        poll_interval)
+                # bpo-35017: shutdown() called during select(), exit immediately.
+                if self.__shutdown_request:
+                    break
                 if self in r:
                     self._handle_request_noblock()
         finally:
diff --git a/lib-python/2.7/_pyio.py b/lib-python/2.7/_pyio.py
--- a/lib-python/2.7/_pyio.py
+++ b/lib-python/2.7/_pyio.py
@@ -1619,6 +1619,7 @@
         self.buffer.write(b)
         if self._line_buffering and (haslf or "\r" in s):
             self.flush()
+        self._set_decoded_chars('')
         self._snapshot = None
         if self._decoder:
             self._decoder.reset()
diff --git a/lib-python/2.7/_strptime.py b/lib-python/2.7/_strptime.py
--- a/lib-python/2.7/_strptime.py
+++ b/lib-python/2.7/_strptime.py
@@ -254,8 +254,8 @@
         # format directives (%m, etc.).
         regex_chars = re_compile(r"([\\.^$*+?\(\){}\[\]|])")
         format = regex_chars.sub(r"\\\1", format)
-        whitespace_replacement = re_compile('\s+')
-        format = whitespace_replacement.sub('\s+', format)
+        whitespace_replacement = re_compile(r'\s+')
+        format = whitespace_replacement.sub(r'\\s+', format)
         while '%' in format:
             directive_index = format.index('%')+1
             processed_format = "%s%s%s" % (processed_format,
diff --git a/lib-python/2.7/_threading_local.py b/lib-python/2.7/_threading_local.py
--- a/lib-python/2.7/_threading_local.py
+++ b/lib-python/2.7/_threading_local.py
@@ -57,11 +57,7 @@
 
   >>> class MyLocal(local):
   ...     number = 2
-  ...     initialized = False
   ...     def __init__(self, **kw):
-  ...         if self.initialized:
-  ...             raise SystemError('__init__ called too many times')
-  ...         self.initialized = True
   ...         self.__dict__.update(kw)
   ...     def squared(self):
   ...         return self.number ** 2
@@ -98,7 +94,7 @@
   >>> thread.start()
   >>> thread.join()
   >>> log
-  [[('color', 'red'), ('initialized', True)], 11]
+  [[('color', 'red')], 11]
 
 without affecting this thread's data:
 
diff --git a/lib-python/2.7/aifc.py b/lib-python/2.7/aifc.py
--- a/lib-python/2.7/aifc.py
+++ b/lib-python/2.7/aifc.py
@@ -288,6 +288,8 @@
     # _ssnd_chunk -- instantiation of a chunk class for the SSND chunk
     # _framesize -- size of one frame in the file
 
+    _file = None  # Set here since __del__ checks it
+
     def initfp(self, file):
         self._version = 0
         self._decomp = None
@@ -306,6 +308,7 @@
         else:
             raise Error, 'not an AIFF or AIFF-C file'
         self._comm_chunk_read = 0
+        self._ssnd_chunk = None
         while 1:
             self._ssnd_seek_needed = 1
             try:
@@ -341,10 +344,16 @@
             self._decomp.SetParams(params)
 
     def __init__(self, f):
-        if type(f) == type(''):
+        if isinstance(f, basestring):
             f = __builtin__.open(f, 'rb')
-        # else, assume it is an open file object already
-        self.initfp(f)
+            try:
+                self.initfp(f)
+            except:
+                f.close()
+                raise
+        else:
+            # assume it is an open file object already
+            self.initfp(f)
 
     #
     # User visible methods.
@@ -562,8 +571,10 @@
     # _datalength -- the size of the audio samples written to the header
     # _datawritten -- the size of the audio samples actually written
 
+    _file = None  # Set here since __del__ checks it
+
     def __init__(self, f):
-        if type(f) == type(''):
+        if isinstance(f, basestring):
             filename = f
             f = __builtin__.open(f, 'wb')
         else:
diff --git a/lib-python/2.7/argparse.py b/lib-python/2.7/argparse.py
--- a/lib-python/2.7/argparse.py
+++ b/lib-python/2.7/argparse.py
@@ -324,7 +324,11 @@
             if len(prefix) + len(usage) > text_width:
 
                 # break usage into wrappable parts
-                part_regexp = r'\(.*?\)+|\[.*?\]+|\S+'
+                part_regexp = (
+                    r'\(.*?\)+(?=\s|$)|'
+                    r'\[.*?\]+(?=\s|$)|'
+                    r'\S+'
+                )
                 opt_usage = format(optionals, groups)
                 pos_usage = format(positionals, groups)
                 opt_parts = _re.findall(part_regexp, opt_usage)
diff --git a/lib-python/2.7/asynchat.py b/lib-python/2.7/asynchat.py
--- a/lib-python/2.7/asynchat.py
+++ b/lib-python/2.7/asynchat.py
@@ -133,7 +133,7 @@
                 # no terminator, collect it all
                 self.collect_incoming_data (self.ac_in_buffer)
                 self.ac_in_buffer = ''
-            elif isinstance(terminator, int) or isinstance(terminator, long):
+            elif isinstance(terminator, (int, long)):
                 # numeric terminator
                 n = terminator
                 if lb < n:
diff --git a/lib-python/2.7/asyncore.py b/lib-python/2.7/asyncore.py
--- a/lib-python/2.7/asyncore.py
+++ b/lib-python/2.7/asyncore.py
@@ -633,7 +633,11 @@
         write = send
 
         def close(self):
-            os.close(self.fd)
+            if self.fd < 0:
+                return
+            fd = self.fd
+            self.fd = -1
+            os.close(fd)
 
         def fileno(self):
             return self.fd
diff --git a/lib-python/2.7/bsddb/test/test_associate.py b/lib-python/2.7/bsddb/test/test_associate.py
--- a/lib-python/2.7/bsddb/test/test_associate.py
+++ b/lib-python/2.7/bsddb/test/test_associate.py
@@ -114,6 +114,22 @@
             dupDB.close()
             self.fail("DBError exception was expected")
 
+    @unittest.skipUnless(db.version() >= (4, 6), 'Needs 4.6+')
+    def test_associateListError(self):
+        db1 = db.DB(self.env)
+        db1.open('bad.db', "a.db", db.DB_BTREE, db.DB_CREATE)
+        db2 = db.DB(self.env)
+        db2.open('bad.db', "b.db", db.DB_BTREE, db.DB_CREATE)
+
+        db1.associate(db2, lambda a, b: [0])
+
+        msg = "TypeError: The list returned by DB->associate callback" \
+              " should be a list of strings."
+        with test_support.captured_output("stderr") as s:
+            db1.put("0", "1")
+        db1.close()
+        db2.close()
+        self.assertEquals(s.getvalue().strip(), msg)
 
 
 #----------------------------------------------------------------------
@@ -233,7 +249,7 @@
         self.assertEqual(vals, None, vals)
 
         vals = secDB.pget('Unknown', txn=txn)
-        self.assertTrue(vals[0] == 99 or vals[0] == '99', vals)
+        self.assertIn(vals[0], (99, '99'), vals)
         vals[1].index('Unknown')
         vals[1].index('Unnamed')
         vals[1].index('unknown')
@@ -247,7 +263,8 @@
             if type(self.keytype) == type(''):
                 self.assertTrue(int(rec[0]))  # for primary db, key is a number
             else:
-                self.assertTrue(rec[0] and type(rec[0]) == type(0))
+                self.assertTrue(rec[0])
+                self.assertIs(type(rec[0]), int)
             count = count + 1
             if verbose:
                 print rec
@@ -262,7 +279,7 @@
 
         # test cursor pget
         vals = self.cur.pget('Unknown', flags=db.DB_LAST)
-        self.assertTrue(vals[1] == 99 or vals[1] == '99', vals)
+        self.assertIn(vals[1], (99, '99'), vals)
         self.assertEqual(vals[0], 'Unknown')
         vals[2].index('Unknown')
         vals[2].index('Unnamed')
diff --git a/lib-python/2.7/bsddb/test/test_basics.py b/lib-python/2.7/bsddb/test/test_basics.py
--- a/lib-python/2.7/bsddb/test/test_basics.py
+++ b/lib-python/2.7/bsddb/test/test_basics.py
@@ -597,7 +597,7 @@
 
         d.put("abcde", "ABCDE");
         num = d.truncate()
-        self.assertTrue(num >= 1, "truncate returned <= 0 on non-empty database")
+        self.assertGreaterEqual(num, 1, "truncate returned <= 0 on non-empty database")
         num = d.truncate()
         self.assertEqual(num, 0,
                 "truncate on empty DB returned nonzero (%r)" % (num,))
@@ -616,9 +616,9 @@
     if db.version() >= (4, 6):
         def test08_exists(self) :
             self.d.put("abcde", "ABCDE")
-            self.assertTrue(self.d.exists("abcde") == True,
+            self.assertEqual(self.d.exists("abcde"), True,
                     "DB->exists() returns wrong value")
-            self.assertTrue(self.d.exists("x") == False,
+            self.assertEqual(self.d.exists("x"), False,
                     "DB->exists() returns wrong value")
 
     #----------------------------------------
@@ -773,7 +773,7 @@
             if verbose:
                 print 'log file: ' + log
             logs = self.env.log_archive(db.DB_ARCH_REMOVE)
-            self.assertTrue(not logs)
+            self.assertFalse(logs)
 
         self.txn = self.env.txn_begin()
 
@@ -785,9 +785,9 @@
             self.d.put("abcde", "ABCDE", txn=txn)
             txn.commit()
             txn = self.env.txn_begin()
-            self.assertTrue(self.d.exists("abcde", txn=txn) == True,
+            self.assertEqual(self.d.exists("abcde", txn=txn), True,
                     "DB->exists() returns wrong value")
-            self.assertTrue(self.d.exists("x", txn=txn) == False,
+            self.assertEqual(self.d.exists("x", txn=txn), False,
                     "DB->exists() returns wrong value")
             txn.abort()
 
@@ -802,7 +802,7 @@
         d.put("abcde", "ABCDE");
         txn = self.env.txn_begin()
         num = d.truncate(txn)
-        self.assertTrue(num >= 1, "truncate returned <= 0 on non-empty database")
+        self.assertGreaterEqual(num, 1, "truncate returned <= 0 on non-empty database")
         num = d.truncate(txn)
         self.assertEqual(num, 0,
                 "truncate on empty DB returned nonzero (%r)" % (num,))
@@ -1086,7 +1086,7 @@
         a = "example of private object"
         self.obj.set_private(a)
         b = self.obj.get_private()
-        self.assertTrue(a is b)  # Object identity
+        self.assertIs(a, b)  # Object identity
 
     def test03_leak_assignment(self) :
         a = "example of private object"
diff --git a/lib-python/2.7/bsddb/test/test_dbenv.py b/lib-python/2.7/bsddb/test/test_dbenv.py
--- a/lib-python/2.7/bsddb/test/test_dbenv.py
+++ b/lib-python/2.7/bsddb/test/test_dbenv.py
@@ -54,15 +54,15 @@
                 self.env.set_cache_max(0, size)
                 size2 = self.env.get_cache_max()
                 self.assertEqual(0, size2[0])
-                self.assertTrue(size <= size2[1])
-                self.assertTrue(2*size > size2[1])
+                self.assertLessEqual(size, size2[1])
+                self.assertGreater(2*size, size2[1])
 
     if db.version() >= (4, 4) :
         def test_mutex_stat(self) :
             self.env.open(self.homeDir, db.DB_CREATE | db.DB_INIT_MPOOL |
                     db.DB_INIT_LOCK)
             stat = self.env.mutex_stat()
-            self.assertTrue("mutex_inuse_max" in stat)
+            self.assertIn("mutex_inuse_max", stat)
 
         def test_lg_filemode(self) :
             for i in [0600, 0660, 0666] :
@@ -128,8 +128,8 @@
                 i = i*1024*1024
                 self.env.set_lg_regionmax(i)
                 j = self.env.get_lg_regionmax()
-                self.assertTrue(i <= j)
-                self.assertTrue(2*i > j)
+                self.assertLessEqual(i, j)
+                self.assertGreater(2*i, j)
 
         def test_lk_detect(self) :
             flags= [db.DB_LOCK_DEFAULT, db.DB_LOCK_EXPIRE, db.DB_LOCK_MAXLOCKS,
@@ -150,10 +150,10 @@
         def test_lg_bsize(self) :
             log_size = 70*1024
             self.env.set_lg_bsize(log_size)
-            self.assertTrue(self.env.get_lg_bsize() >= log_size)
-            self.assertTrue(self.env.get_lg_bsize() < 4*log_size)
+            self.assertGreaterEqual(self.env.get_lg_bsize(), log_size)
+            self.assertLess(self.env.get_lg_bsize(), 4*log_size)
             self.env.set_lg_bsize(4*log_size)
-            self.assertTrue(self.env.get_lg_bsize() >= 4*log_size)
+            self.assertGreaterEqual(self.env.get_lg_bsize(), 4*log_size)
 
         def test_setget_data_dirs(self) :
             dirs = ("a", "b", "c", "d")
@@ -185,7 +185,7 @@
             self.assertEqual(cachesize2[0], cachesize3[0])
             self.assertEqual(cachesize2[2], cachesize3[2])
             # In Berkeley DB 5.1, the cachesize can change when opening the Env
-            self.assertTrue(cachesize2[1] <= cachesize3[1])
+            self.assertLessEqual(cachesize2[1], cachesize3[1])
 
         def test_set_cachesize_dbenv_db(self) :
             # You can not configure the cachesize using
@@ -299,7 +299,7 @@
             msg = "This is a test..."
             self.env.log_printf(msg)
             logc = self.env.log_cursor()
-            self.assertTrue(msg in (logc.last()[1]))
+            self.assertIn(msg, logc.last()[1])
 
     if db.version() >= (4, 7) :
         def test_log_config(self) :
@@ -341,21 +341,21 @@
             txn.commit()
             logc = self.env.log_cursor()
             logc.last()  # Skip the commit
-            self.assertTrue(msg in (logc.prev()[1]))
+            self.assertIn(msg, logc.prev()[1])
 
             msg = "This is another test..."
             txn = self.env.txn_begin()
             self.env.log_printf(msg, txn=txn)
             txn.abort()  # Do not store the new message
             logc.last()  # Skip the abort
-            self.assertTrue(msg not in (logc.prev()[1]))
+            self.assertNotIn(msg, logc.prev()[1])
 
             msg = "This is a third test..."
             txn = self.env.txn_begin()
             self.env.log_printf(msg, txn=txn)
             txn.commit()  # Do not store the new message
             logc.last()  # Skip the commit
-            self.assertTrue(msg in (logc.prev()[1]))
+            self.assertIn(msg, logc.prev()[1])
 
 
 class DBEnv_memp(DBEnv):
@@ -372,39 +372,39 @@
 
     def test_memp_1_trickle(self) :
         self.db.put("hi", "bye")
-        self.assertTrue(self.env.memp_trickle(100) > 0)
+        self.assertGreater(self.env.memp_trickle(100), 0)
 
 # Preserve the order, do "memp_trickle" test first
     def test_memp_2_sync(self) :
         self.db.put("hi", "bye")
         self.env.memp_sync()  # Full flush
         # Nothing to do...
-        self.assertTrue(self.env.memp_trickle(100) == 0)
+        self.assertEqual(self.env.memp_trickle(100), 0)
 
         self.db.put("hi", "bye2")
         self.env.memp_sync((1, 0))  # NOP, probably
         # Something to do... or not
-        self.assertTrue(self.env.memp_trickle(100) >= 0)
+        self.assertGreaterEqual(self.env.memp_trickle(100), 0)
 
         self.db.put("hi", "bye3")
         self.env.memp_sync((123, 99))  # Full flush
         # Nothing to do...
-        self.assertTrue(self.env.memp_trickle(100) == 0)
+        self.assertEqual(self.env.memp_trickle(100), 0)
 
     def test_memp_stat_1(self) :
         stats = self.env.memp_stat()  # No param
-        self.assertTrue(len(stats)==2)
-        self.assertTrue("cache_miss" in stats[0])
+        self.assertEqual(len(stats), 2)
+        self.assertIn("cache_miss", stats[0])
         stats = self.env.memp_stat(db.DB_STAT_CLEAR)  # Positional param
-        self.assertTrue("cache_miss" in stats[0])
+        self.assertIn("cache_miss", stats[0])
         stats = self.env.memp_stat(flags=0)  # Keyword param
-        self.assertTrue("cache_miss" in stats[0])
+        self.assertIn("cache_miss", stats[0])
 
     def test_memp_stat_2(self) :
         stats=self.env.memp_stat()[1]
-        self.assertTrue(len(stats))==1
-        self.assertTrue("test" in stats)
-        self.assertTrue("page_in" in stats["test"])
+        self.assertEqual(len(stats), 1)
+        self.assertIn("test", stats)
+        self.assertIn("page_in", stats["test"])
 
 class DBEnv_logcursor(DBEnv):
     def setUp(self):
@@ -426,28 +426,28 @@
         DBEnv.tearDown(self)
 
     def _check_return(self, value) :
-        self.assertTrue(isinstance(value, tuple))
+        self.assertIsInstance(value, tuple)
         self.assertEqual(len(value), 2)
-        self.assertTrue(isinstance(value[0], tuple))
+        self.assertIsInstance(value[0], tuple)
         self.assertEqual(len(value[0]), 2)
-        self.assertTrue(isinstance(value[0][0], int))
-        self.assertTrue(isinstance(value[0][1], int))
-        self.assertTrue(isinstance(value[1], str))
+        self.assertIsInstance(value[0][0], int)
+        self.assertIsInstance(value[0][1], int)
+        self.assertIsInstance(value[1], str)
 
     # Preserve test order
     def test_1_first(self) :
         logc = self.env.log_cursor()
         v = logc.first()
         self._check_return(v)
-        self.assertTrue((1, 1) < v[0])
-        self.assertTrue(len(v[1])>0)
+        self.assertLess((1, 1), v[0])
+        self.assertGreater(len(v[1]), 0)
 
     def test_2_last(self) :
         logc = self.env.log_cursor()
         lsn_first = logc.first()[0]
         v = logc.last()
         self._check_return(v)
-        self.assertTrue(lsn_first < v[0])
+        self.assertLess(lsn_first, v[0])
 
     def test_3_next(self) :
         logc = self.env.log_cursor()
@@ -456,16 +456,16 @@
         lsn_first = logc.first()[0]
         v = logc.next()
         self._check_return(v)
-        self.assertTrue(lsn_first < v[0])
-        self.assertTrue(lsn_last > v[0])
+        self.assertLess(lsn_first, v[0])
+        self.assertGreater(lsn_last, v[0])
 
         v2 = logc.next()
-        self.assertTrue(v2[0] > v[0])
-        self.assertTrue(lsn_last > v2[0])
+        self.assertGreater(v2[0], v[0])
+        self.assertGreater(lsn_last, v2[0])
 
         v3 = logc.next()
-        self.assertTrue(v3[0] > v2[0])
-        self.assertTrue(lsn_last > v3[0])
+        self.assertGreater(v3[0], v2[0])
+        self.assertGreater(lsn_last, v3[0])
 
     def test_4_prev(self) :
         logc = self.env.log_cursor()
@@ -474,16 +474,16 @@
         lsn_last = logc.last()[0]
         v = logc.prev()
         self._check_return(v)
-        self.assertTrue(lsn_first < v[0])
-        self.assertTrue(lsn_last > v[0])
+        self.assertLess(lsn_first, v[0])
+        self.assertGreater(lsn_last, v[0])
 
         v2 = logc.prev()
-        self.assertTrue(v2[0] < v[0])
-        self.assertTrue(lsn_first < v2[0])
+        self.assertLess(v2[0], v[0])
+        self.assertLess(lsn_first, v2[0])
 
         v3 = logc.prev()
-        self.assertTrue(v3[0] < v2[0])
-        self.assertTrue(lsn_first < v3[0])
+        self.assertLess(v3[0], v2[0])
+        self.assertLess(lsn_first, v3[0])
 
     def test_5_current(self) :
         logc = self.env.log_cursor()
diff --git a/lib-python/2.7/bsddb/test/test_dbshelve.py b/lib-python/2.7/bsddb/test/test_dbshelve.py
--- a/lib-python/2.7/bsddb/test/test_dbshelve.py
+++ b/lib-python/2.7/bsddb/test/test_dbshelve.py
@@ -248,7 +248,7 @@
             self.assertEqual(value.L, [x] * 10)
 
         else:
-            self.assertTrue(0, 'Unknown key type, fix the test')
+            self.fail('Unknown key type, fix the test')
 
 #----------------------------------------------------------------------
 
diff --git a/lib-python/2.7/bsddb/test/test_dbtables.py b/lib-python/2.7/bsddb/test/test_dbtables.py
--- a/lib-python/2.7/bsddb/test/test_dbtables.py
+++ b/lib-python/2.7/bsddb/test/test_dbtables.py
@@ -82,8 +82,8 @@
             colval = pickle.loads(values[0][colname])
         else :
             colval = pickle.loads(bytes(values[0][colname], "iso8859-1"))
-        self.assertTrue(colval > 3.141)
-        self.assertTrue(colval < 3.142)
+        self.assertGreater(colval, 3.141)
+        self.assertLess(colval, 3.142)
 
 
     def test02(self):
diff --git a/lib-python/2.7/bsddb/test/test_distributed_transactions.py b/lib-python/2.7/bsddb/test/test_distributed_transactions.py
--- a/lib-python/2.7/bsddb/test/test_distributed_transactions.py
+++ b/lib-python/2.7/bsddb/test/test_distributed_transactions.py
@@ -79,7 +79,7 @@
         recovered_txns=self.dbenv.txn_recover()
         self.assertEqual(self.num_txns,len(recovered_txns))
         for gid,txn in recovered_txns :
-            self.assertTrue(gid in txns)
+            self.assertIn(gid, txns)
         del txn
         del recovered_txns
 
@@ -122,7 +122,7 @@
     # Be sure there are not pending transactions.
     # Check also database size.
         recovered_txns=self.dbenv.txn_recover()
-        self.assertTrue(len(recovered_txns)==0)
+        self.assertEqual(len(recovered_txns), 0)
         self.assertEqual(len(committed_txns),self.db.stat()["nkeys"])
 
 class DBTxn_distributedSYNC(DBTxn_distributed):
diff --git a/lib-python/2.7/bsddb/test/test_lock.py b/lib-python/2.7/bsddb/test/test_lock.py
--- a/lib-python/2.7/bsddb/test/test_lock.py
+++ b/lib-python/2.7/bsddb/test/test_lock.py
@@ -2,6 +2,7 @@
 TestCases for testing the locking sub-system.
 """
 
+import sys
 import time
 
 import unittest
@@ -10,7 +11,6 @@
 
 if have_threads :
     from threading import Thread
-    import sys
     if sys.version_info[0] < 3 :
         from threading import currentThread
     else :
@@ -129,7 +129,14 @@
         end_time=time.time()
         deadlock_detection.end=True
         # Floating point rounding
-        self.assertTrue((end_time-start_time) >= 0.0999)
+        if sys.platform == 'win32':
+            # bpo-30850: On Windows, tolerate 50 ms whereas 100 ms is expected.
+            # The lock sometimes times out after only 58 ms. Windows clocks
+            # have a bad resolution and bad accuracy.
+            min_dt = 0.050
+        else:
+            min_dt = 0.0999
+        self.assertGreaterEqual(end_time-start_time, min_dt)
         self.env.lock_put(lock)
         t.join()
 
@@ -137,7 +144,7 @@
         self.env.lock_id_free(anID2)
 
         if db.version() >= (4,6):
-            self.assertTrue(deadlock_detection.count>0)
+            self.assertGreater(deadlock_detection.count, 0)
 
     def theThread(self, lockType):
         import sys
diff --git a/lib-python/2.7/bsddb/test/test_misc.py b/lib-python/2.7/bsddb/test/test_misc.py
--- a/lib-python/2.7/bsddb/test/test_misc.py
+++ b/lib-python/2.7/bsddb/test/test_misc.py
@@ -25,7 +25,7 @@
     def test02_db_home(self):
         env = db.DBEnv()
         # check for crash fixed when db_home is used before open()
-        self.assertTrue(env.db_home is None)
+        self.assertIsNone(env.db_home)
         env.open(self.homeDir, db.DB_CREATE)
         if sys.version_info[0] < 3 :
             self.assertEqual(self.homeDir, env.db_home)
diff --git a/lib-python/2.7/bsddb/test/test_recno.py b/lib-python/2.7/bsddb/test/test_recno.py
--- a/lib-python/2.7/bsddb/test/test_recno.py
+++ b/lib-python/2.7/bsddb/test/test_recno.py
@@ -18,7 +18,7 @@
         def assertIsInstance(self, obj, datatype, msg=None) :
             return self.assertEqual(type(obj), datatype, msg=msg)
         def assertGreaterEqual(self, a, b, msg=None) :
-            return self.assertTrue(a>=b, msg=msg)
+            return self.assertGreaterEqual(a, b, msg=msg)
 
 
     def setUp(self):
diff --git a/lib-python/2.7/bsddb/test/test_replication.py b/lib-python/2.7/bsddb/test/test_replication.py
--- a/lib-python/2.7/bsddb/test/test_replication.py
+++ b/lib-python/2.7/bsddb/test/test_replication.py
@@ -186,20 +186,18 @@
         d = d.values()[0]  # There is only one
         self.assertEqual(d[0], "127.0.0.1")
         self.assertEqual(d[1], client_port)
-        self.assertTrue((d[2]==db.DB_REPMGR_CONNECTED) or \
-                (d[2]==db.DB_REPMGR_DISCONNECTED))
+        self.assertIn(d[2], (db.DB_REPMGR_CONNECTED, db.DB_REPMGR_DISCONNECTED))
 
         d = self.dbenvClient.repmgr_site_list()
         self.assertEqual(len(d), 1)
         d = d.values()[0]  # There is only one
         self.assertEqual(d[0], "127.0.0.1")
         self.assertEqual(d[1], master_port)
-        self.assertTrue((d[2]==db.DB_REPMGR_CONNECTED) or \
-                (d[2]==db.DB_REPMGR_DISCONNECTED))
+        self.assertIn(d[2], (db.DB_REPMGR_CONNECTED, db.DB_REPMGR_DISCONNECTED))
 
         if db.version() >= (4,6) :
             d = self.dbenvMaster.repmgr_stat(flags=db.DB_STAT_CLEAR);
-            self.assertTrue("msgs_queued" in d)
+            self.assertIn("msgs_queued", d)
 
         self.dbMaster=db.DB(self.dbenvMaster)
         txn=self.dbenvMaster.txn_begin()
@@ -247,7 +245,7 @@
         if time.time()>=timeout and startup_timeout:
             self.skipTest("replication test skipped due to random failure, "
                 "see issue 3892")
-        self.assertTrue(time.time()<timeout)
+        self.assertLess(time.time(), timeout)
         self.assertEqual("123", v)
 
         txn=self.dbenvMaster.txn_begin()
@@ -260,7 +258,7 @@
             txn.commit()
             if v is None :
                 time.sleep(0.02)
-        self.assertTrue(time.time()<timeout)
+        self.assertLess(time.time(), timeout)
         self.assertEqual(None, v)
 
 class DBBaseReplication(DBReplication) :
@@ -381,7 +379,7 @@
         while (time.time()<timeout) and not (self.confirmed_master and
                 self.client_startupdone) :
             time.sleep(0.02)
-        self.assertTrue(time.time()<timeout)
+        self.assertLess(time.time(), timeout)
 
         self.dbMaster=db.DB(self.dbenvMaster)
         txn=self.dbenvMaster.txn_begin()
@@ -410,7 +408,7 @@
             break
 
         d = self.dbenvMaster.rep_stat(flags=db.DB_STAT_CLEAR);
-        self.assertTrue("master_changes" in d)
+        self.assertIn("master_changes", d)
 
         txn=self.dbenvMaster.txn_begin()
         self.dbMaster.put("ABC", "123", txn=txn)
@@ -424,7 +422,7 @@
             txn.commit()
             if v is None :
                 time.sleep(0.02)
-        self.assertTrue(time.time()<timeout)
+        self.assertLess(time.time(), timeout)
         self.assertEqual("123", v)
 
         txn=self.dbenvMaster.txn_begin()
@@ -437,7 +435,7 @@
             txn.commit()
             if v is None :
                 time.sleep(0.02)
-        self.assertTrue(time.time()<timeout)
+        self.assertLess(time.time(), timeout)
         self.assertEqual(None, v)
 
     if db.version() >= (4,7) :
diff --git a/lib-python/2.7/bsddb/test/test_sequence.py b/lib-python/2.7/bsddb/test/test_sequence.py
--- a/lib-python/2.7/bsddb/test/test_sequence.py
+++ b/lib-python/2.7/bsddb/test/test_sequence.py
@@ -82,7 +82,7 @@
         stat = self.seq.stat()
         for param in ('nowait', 'min', 'max', 'value', 'current',
                       'flags', 'cache_size', 'last_value', 'wait'):
-            self.assertTrue(param in stat, "parameter %s isn't in stat info" % param)
+            self.assertIn(param, stat, "parameter %s isn't in stat info" % param)
 
     if db.version() >= (4,7) :
         # This code checks a crash solved in Berkeley DB 4.7
diff --git a/lib-python/2.7/bsddb/test/test_thread.py b/lib-python/2.7/bsddb/test/test_thread.py
--- a/lib-python/2.7/bsddb/test/test_thread.py
+++ b/lib-python/2.7/bsddb/test/test_thread.py
@@ -85,7 +85,7 @@
         readers_per_writer=self.readers//self.writers
         self.assertEqual(self.records,self.writers*records_per_writer)
         self.assertEqual(self.readers,self.writers*readers_per_writer)
-        self.assertTrue((records_per_writer%readers_per_writer)==0)
+        self.assertEqual(records_per_writer%readers_per_writer, 0)
         readers = []
 
         for x in xrange(self.readers):
@@ -213,7 +213,7 @@
         readers_per_writer=self.readers//self.writers
         self.assertEqual(self.records,self.writers*records_per_writer)
         self.assertEqual(self.readers,self.writers*readers_per_writer)
-        self.assertTrue((records_per_writer%readers_per_writer)==0)
+        self.assertEqual(records_per_writer%readers_per_writer, 0)
 
         readers = []
         for x in xrange(self.readers):
@@ -339,7 +339,7 @@
         readers_per_writer=self.readers//self.writers
         self.assertEqual(self.records,self.writers*records_per_writer)
         self.assertEqual(self.readers,self.writers*readers_per_writer)
-        self.assertTrue((records_per_writer%readers_per_writer)==0)
+        self.assertEqual(records_per_writer%readers_per_writer, 0)
 
         readers=[]
         for x in xrange(self.readers):
diff --git a/lib-python/2.7/cProfile.py b/lib-python/2.7/cProfile.py
--- a/lib-python/2.7/cProfile.py
+++ b/lib-python/2.7/cProfile.py
@@ -64,11 +64,11 @@
 # ____________________________________________________________
 
 class Profile(_lsprof.Profiler):
-    """Profile(custom_timer=None, time_unit=None, subcalls=True, builtins=True)
+    """Profile(timer=None, timeunit=None, subcalls=True, builtins=True)
 
     Builds a profiler object using the specified timer function.
     The default timer is a fast built-in one based on real time.
-    For custom timer functions returning integers, time_unit can
+    For custom timer functions returning integers, timeunit can
     be a float specifying a scale (i.e. how long each integer unit
     is, in seconds).
     """
@@ -161,7 +161,7 @@
 # ____________________________________________________________
 
 def main():
-    import os, sys, types
+    import os, sys, pstats, types
     from optparse import OptionParser
     usage = "cProfile.py [-o output_file_path] [-s sort] scriptfile [arg] ..."
     parser = OptionParser(usage=usage)
@@ -170,7 +170,8 @@
         help="Save stats to <outfile>", default=None)
     parser.add_option('-s', '--sort', dest="sort",
         help="Sort order when printing to stdout, based on pstats.Stats class",
-        default=-1)
+        default=-1,
+        choices=sorted(pstats.Stats.sort_arg_dict_default))
 
     if not sys.argv[1:]:
         parser.print_usage()
diff --git a/lib-python/2.7/cgi.py b/lib-python/2.7/cgi.py
old mode 100755
new mode 100644
--- a/lib-python/2.7/cgi.py
+++ b/lib-python/2.7/cgi.py
@@ -184,11 +184,12 @@
     return urlparse.parse_qs(qs, keep_blank_values, strict_parsing)
 
 
-def parse_qsl(qs, keep_blank_values=0, strict_parsing=0):
+def parse_qsl(qs, keep_blank_values=0, strict_parsing=0, max_num_fields=None):
     """Parse a query given as a string argument."""
     warn("cgi.parse_qsl is deprecated, use urlparse.parse_qsl instead",
          PendingDeprecationWarning, 2)
-    return urlparse.parse_qsl(qs, keep_blank_values, strict_parsing)
+    return urlparse.parse_qsl(qs, keep_blank_values, strict_parsing,
+                              max_num_fields)
 
 def parse_multipart(fp, pdict):
     """Parse multipart input.
@@ -393,7 +394,8 @@
     """
 
     def __init__(self, fp=None, headers=None, outerboundary="",
-                 environ=os.environ, keep_blank_values=0, strict_parsing=0):
+                 environ=os.environ, keep_blank_values=0, strict_parsing=0,
+                 max_num_fields=None):
         """Constructor.  Read multipart/* until last part.
 
         Arguments, all optional:
@@ -420,10 +422,14 @@
             If false (the default), errors are silently ignored.
             If true, errors raise a ValueError exception.
 
+        max_num_fields: int. If set, then __init__ throws a ValueError
+            if there are more than n fields read by parse_qsl().
+
         """
         method = 'GET'
         self.keep_blank_values = keep_blank_values
         self.strict_parsing = strict_parsing
+        self.max_num_fields = max_num_fields
         if 'REQUEST_METHOD' in environ:
             method = environ['REQUEST_METHOD'].upper()
         self.qs_on_post = None
@@ -606,10 +612,9 @@
         qs = self.fp.read(self.length)
         if self.qs_on_post:
             qs += '&' + self.qs_on_post
-        self.list = list = []
-        for key, value in urlparse.parse_qsl(qs, self.keep_blank_values,
-                                            self.strict_parsing):
-            list.append(MiniFieldStorage(key, value))
+        query = urlparse.parse_qsl(qs, self.keep_blank_values,
+                                   self.strict_parsing, self.max_num_fields)
+        self.list = [MiniFieldStorage(key, value) for key, value in query]
         self.skip_lines()
 
     FieldStorageClass = None
@@ -621,19 +626,38 @@
             raise ValueError, 'Invalid boundary in multipart form: %r' % (ib,)
         self.list = []
         if self.qs_on_post:
-            for key, value in urlparse.parse_qsl(self.qs_on_post,
-                                self.keep_blank_values, self.strict_parsing):
-                self.list.append(MiniFieldStorage(key, value))
+            query = urlparse.parse_qsl(self.qs_on_post,
+                                       self.keep_blank_values,
+                                       self.strict_parsing,
+                                       self.max_num_fields)
+            self.list.extend(MiniFieldStorage(key, value)
+                             for key, value in query)
             FieldStorageClass = None
 
+        # Propagate max_num_fields into the sub class appropriately
+        max_num_fields = self.max_num_fields
+        if max_num_fields is not None:
+            max_num_fields -= len(self.list)
+
         klass = self.FieldStorageClass or self.__class__
         part = klass(self.fp, {}, ib,
-                     environ, keep_blank_values, strict_parsing)
+                     environ, keep_blank_values, strict_parsing,
+                     max_num_fields)
+
         # Throw first part away
         while not part.done:
             headers = rfc822.Message(self.fp)
             part = klass(self.fp, headers, ib,
-                         environ, keep_blank_values, strict_parsing)
+                         environ, keep_blank_values, strict_parsing,
+                         max_num_fields)
+
+            if max_num_fields is not None:
+                max_num_fields -= 1
+                if part.list:
+                    max_num_fields -= len(part.list)
+                if max_num_fields < 0:
+                    raise ValueError('Max number of fields exceeded')
+
             self.list.append(part)
         self.skip_lines()
 
diff --git a/lib-python/2.7/cgitb.py b/lib-python/2.7/cgitb.py
--- a/lib-python/2.7/cgitb.py
+++ b/lib-python/2.7/cgitb.py
@@ -125,7 +125,7 @@
         args, varargs, varkw, locals = inspect.getargvalues(frame)
         call = ''
         if func != '?':
-            call = 'in ' + strong(func) + \
+            call = 'in ' + strong(pydoc.html.escape(func)) + \
                 inspect.formatargvalues(args, varargs, varkw, locals,
                     formatvalue=lambda value: '=' + pydoc.html.repr(value))
 
@@ -285,7 +285,7 @@
 
         if self.display:
             if plain:
-                doc = doc.replace('&', '&amp;').replace('<', '&lt;')
+                doc = pydoc.html.escape(doc)
                 self.file.write('<pre>' + doc + '</pre>\n')
             else:
                 self.file.write(doc + '\n')
diff --git a/lib-python/2.7/codecs.py b/lib-python/2.7/codecs.py
--- a/lib-python/2.7/codecs.py
+++ b/lib-python/2.7/codecs.py
@@ -472,15 +472,17 @@
             self.charbuffer = "".join(self.linebuffer)
             self.linebuffer = None
 
+        if chars < 0:
+            # For compatibility with other read() methods that take a
+            # single argument
+            chars = size
+
         # read until we get the required number of characters (if available)
         while True:
             # can the request be satisfied from the character buffer?
             if chars >= 0:
                 if len(self.charbuffer) >= chars:
                     break
-            elif size >= 0:
-                if len(self.charbuffer) >= size:
-                    break
             # we need more data
             if size < 0:
                 newdata = self.stream.read()
diff --git a/lib-python/2.7/compiler/pyassem.py b/lib-python/2.7/compiler/pyassem.py
--- a/lib-python/2.7/compiler/pyassem.py
+++ b/lib-python/2.7/compiler/pyassem.py
@@ -581,7 +581,7 @@
 
 def twobyte(val):
     """Convert an int argument into high and low bytes"""
-    assert isinstance(val, int)
+    assert isinstance(val, (int, long))
     return divmod(val, 256)
 
 class LineAddrTable:
diff --git a/lib-python/2.7/compiler/transformer.py b/lib-python/2.7/compiler/transformer.py
--- a/lib-python/2.7/compiler/transformer.py
+++ b/lib-python/2.7/compiler/transformer.py
@@ -1526,7 +1526,7 @@
 def debug_tree(tree):
     l = []
     for elt in tree:
-        if isinstance(elt, int):
+        if isinstance(elt, (int, long)):
             l.append(_names.get(elt, elt))
         elif isinstance(elt, str):
             l.append(elt)
diff --git a/lib-python/2.7/copy_reg.py b/lib-python/2.7/copy_reg.py
--- a/lib-python/2.7/copy_reg.py
+++ b/lib-python/2.7/copy_reg.py
@@ -127,7 +127,11 @@
                         continue
                     # mangled names
                     elif name.startswith('__') and not name.endswith('__'):
-                        names.append('_%s%s' % (c.__name__, name))
+                        stripped = c.__name__.lstrip('_')
+                        if stripped:
+                            names.append('_%s%s' % (stripped, name))
+                        else:
+                            names.append(name)
                     else:
                         names.append(name)
 
diff --git a/lib-python/2.7/csv.py b/lib-python/2.7/csv.py
--- a/lib-python/2.7/csv.py
+++ b/lib-python/2.7/csv.py
@@ -217,7 +217,7 @@
         matches = []
         for restr in ('(?P<delim>[^\w\n"\'])(?P<space> ?)(?P<quote>["\']).*?(?P=quote)(?P=delim)', # ,".*?",
                       '(?:^|\n)(?P<quote>["\']).*?(?P=quote)(?P<delim>[^\w\n"\'])(?P<space> ?)',   #  ".*?",
-                      '(?P<delim>>[^\w\n"\'])(?P<space> ?)(?P<quote>["\']).*?(?P=quote)(?:$|\n)',  # ,".*?"
+                      '(?P<delim>[^\w\n"\'])(?P<space> ?)(?P<quote>["\']).*?(?P=quote)(?:$|\n)',   # ,".*?"
                       '(?:^|\n)(?P<quote>["\']).*?(?P=quote)(?:$|\n)'):                            #  ".*?" (no delim, no space)
             regexp = re.compile(restr, re.DOTALL | re.MULTILINE)
             matches = regexp.findall(data)
diff --git a/lib-python/2.7/ctypes/__init__.py b/lib-python/2.7/ctypes/__init__.py
--- a/lib-python/2.7/ctypes/__init__.py
+++ b/lib-python/2.7/ctypes/__init__.py
@@ -343,6 +343,10 @@
     """
     _func_flags_ = _FUNCFLAG_CDECL
     _func_restype_ = c_int
+    # default values for repr
+    _name = '<uninitialized>'
+    _handle = 0
+    _FuncPtr = None
 
     def __init__(self, name, mode=DEFAULT_MODE, handle=None,
                  use_errno=False,
diff --git a/lib-python/2.7/ctypes/test/test_anon.py b/lib-python/2.7/ctypes/test/test_anon.py
--- a/lib-python/2.7/ctypes/test/test_anon.py
+++ b/lib-python/2.7/ctypes/test/test_anon.py
@@ -1,4 +1,5 @@
 import unittest
+from test.support import cpython_only
 from ctypes import *
 
 class AnonTest(unittest.TestCase):
@@ -35,6 +36,18 @@
                                                       {"_fields_": [],
                                                        "_anonymous_": ["x"]}))
 
+    @cpython_only
+    def test_issue31490(self):
+        # There shouldn't be an assertion failure in case the class has an
+        # attribute whose name is specified in _anonymous_ but not in _fields_.
+
+        # AttributeError: 'x' is specified in _anonymous_ but not in _fields_
+        with self.assertRaises(AttributeError):
+            class Name(Structure):
+                _fields_ = []
+                _anonymous_ = ["x"]
+                x = 42
+
     def test_nested(self):
         class ANON_S(Structure):
             _fields_ = [("a", c_int)]
diff --git a/lib-python/2.7/ctypes/test/test_arrays.py b/lib-python/2.7/ctypes/test/test_arrays.py
--- a/lib-python/2.7/ctypes/test/test_arrays.py
+++ b/lib-python/2.7/ctypes/test/test_arrays.py
@@ -1,4 +1,6 @@
 import unittest
+from test.support import precisionbigmemtest, _2G
+import sys
 from ctypes import *
 from test.test_support import impl_detail
 
@@ -143,5 +145,10 @@
         t2 = my_int * 1
         self.assertIs(t1, t2)
 
+    @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform')
+    @precisionbigmemtest(size=_2G, memuse=1, dry_run=False)
+    def test_large_array(self, size):
+        a = c_char * size
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_as_parameter.py b/lib-python/2.7/ctypes/test/test_as_parameter.py
--- a/lib-python/2.7/ctypes/test/test_as_parameter.py
+++ b/lib-python/2.7/ctypes/test/test_as_parameter.py
@@ -24,7 +24,7 @@
         f.argtypes = [c_byte, c_wchar, c_int, c_long, c_float, c_double]
         result = f(self.wrap(1), self.wrap(u"x"), self.wrap(3), self.wrap(4), self.wrap(5.0), self.wrap(6.0))
         self.assertEqual(result, 139)
-        self.assertTrue(type(result), int)
+        self.assertIs(type(result), int)
 
     def test_pointers(self):
         f = dll._testfunc_p_p
diff --git a/lib-python/2.7/ctypes/test/test_callbacks.py b/lib-python/2.7/ctypes/test/test_callbacks.py
--- a/lib-python/2.7/ctypes/test/test_callbacks.py
+++ b/lib-python/2.7/ctypes/test/test_callbacks.py
@@ -252,6 +252,7 @@
     def test_callback_large_struct(self):
         class Check: pass
 
+        # This should mirror the structure in Modules/_ctypes/_ctypes_test.c
         class X(Structure):
             _fields_ = [
                 ('first', c_ulong),
@@ -263,6 +264,11 @@
             check.first = s.first
             check.second = s.second
             check.third = s.third
+            # See issue #29565.
+            # The structure should be passed by value, so
+            # any changes to it should not be reflected in
+            # the value passed
+            s.first = s.second = s.third = 0x0badf00d
 
         check = Check()
         s = X()
@@ -283,6 +289,11 @@
         self.assertEqual(check.first, 0xdeadbeef)
         self.assertEqual(check.second, 0xcafebabe)
         self.assertEqual(check.third, 0x0bad1dea)
+        # See issue #29565.
+        # Ensure that the original struct is unchanged.
+        self.assertEqual(s.first, check.first)
+        self.assertEqual(s.second, check.second)
+        self.assertEqual(s.third, check.third)
 
 ################################################################
 
diff --git a/lib-python/2.7/ctypes/test/test_frombuffer.py b/lib-python/2.7/ctypes/test/test_frombuffer.py
--- a/lib-python/2.7/ctypes/test/test_frombuffer.py
+++ b/lib-python/2.7/ctypes/test/test_frombuffer.py
@@ -78,12 +78,21 @@
                           (c_int * 1).from_buffer_copy, a, 16 * sizeof(c_int))
 
     def test_abstract(self):
+        from ctypes import _Pointer, _SimpleCData, _CFuncPtr
+
         self.assertRaises(TypeError, Array.from_buffer, bytearray(10))
         self.assertRaises(TypeError, Structure.from_buffer, bytearray(10))
         self.assertRaises(TypeError, Union.from_buffer, bytearray(10))
+        self.assertRaises(TypeError, _CFuncPtr.from_buffer, bytearray(10))
+        self.assertRaises(TypeError, _Pointer.from_buffer, bytearray(10))
+        self.assertRaises(TypeError, _SimpleCData.from_buffer, bytearray(10))
+
         self.assertRaises(TypeError, Array.from_buffer_copy, b"123")
         self.assertRaises(TypeError, Structure.from_buffer_copy, b"123")
         self.assertRaises(TypeError, Union.from_buffer_copy, b"123")
+        self.assertRaises(TypeError, _CFuncPtr.from_buffer_copy, b"123")
+        self.assertRaises(TypeError, _Pointer.from_buffer_copy, b"123")
+        self.assertRaises(TypeError, _SimpleCData.from_buffer_copy, b"123")
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_funcptr.py b/lib-python/2.7/ctypes/test/test_funcptr.py
--- a/lib-python/2.7/ctypes/test/test_funcptr.py
+++ b/lib-python/2.7/ctypes/test/test_funcptr.py
@@ -123,5 +123,10 @@
         self.assertEqual(strtok(None, "\n"), "c")
         self.assertEqual(strtok(None, "\n"), None)
 
+    def test_abstract(self):
+        from ctypes import _CFuncPtr
+
+        self.assertRaises(TypeError, _CFuncPtr, 13, "name", 42, "iid")
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_loading.py b/lib-python/2.7/ctypes/test/test_loading.py
--- a/lib-python/2.7/ctypes/test/test_loading.py
+++ b/lib-python/2.7/ctypes/test/test_loading.py
@@ -3,6 +3,7 @@
 import os
 from ctypes.util import find_library
 from ctypes.test import is_resource_enabled, xfail
+import test.test_support as support
 
 libc_name = None
 if os.name == "nt":
@@ -27,6 +28,12 @@
         CDLL(os.path.basename(libc_name))
         self.assertRaises(OSError, CDLL, self.unknowndll)
 
+    @support.requires_unicode
+    @unittest.skipUnless(libc_name is not None, 'could not find libc')
+    def test_load_unicode(self):
+        CDLL(unicode(libc_name))
+        self.assertRaises(OSError, CDLL, unicode(self.unknowndll))
+
     @unittest.skipUnless(libc_name is not None, 'could not find libc')
     @unittest.skipUnless(libc_name is not None and
                          os.path.basename(libc_name) == "libc.so.6",
diff --git a/lib-python/2.7/ctypes/test/test_parameters.py b/lib-python/2.7/ctypes/test/test_parameters.py
--- a/lib-python/2.7/ctypes/test/test_parameters.py
+++ b/lib-python/2.7/ctypes/test/test_parameters.py
@@ -1,5 +1,6 @@
 import unittest, sys
 from ctypes.test import need_symbol
+import test.support
 
 from ctypes.test import xfail
 
@@ -180,6 +181,36 @@
         # ArgumentError: argument 1: ValueError: 99
         self.assertRaises(ArgumentError, func, 99)
 
+    def test_abstract(self):
+        from ctypes import (Array, Structure, Union, _Pointer,
+                            _SimpleCData, _CFuncPtr)
+
+        self.assertRaises(TypeError, Array.from_param, 42)
+        self.assertRaises(TypeError, Structure.from_param, 42)
+        self.assertRaises(TypeError, Union.from_param, 42)
+        self.assertRaises(TypeError, _CFuncPtr.from_param, 42)
+        self.assertRaises(TypeError, _Pointer.from_param, 42)
+        self.assertRaises(TypeError, _SimpleCData.from_param, 42)
+
+    @test.support.cpython_only
+    def test_issue31311(self):
+        # __setstate__ should neither raise a SystemError nor crash in case
+        # of a bad __dict__.
+        from ctypes import Structure
+
+        class BadStruct(Structure):
+            @property
+            def __dict__(self):
+                pass
+        with self.assertRaises(TypeError):
+            BadStruct().__setstate__({}, b'foo')
+
+        class WorseStruct(Structure):
+            @property
+            def __dict__(self):
+                1/0.0
+        with self.assertRaises(ZeroDivisionError):
+            WorseStruct().__setstate__({}, b'foo')
 
 ################################################################
 
diff --git a/lib-python/2.7/ctypes/test/test_pep3118.py b/lib-python/2.7/ctypes/test/test_pep3118.py
--- a/lib-python/2.7/ctypes/test/test_pep3118.py
+++ b/lib-python/2.7/ctypes/test/test_pep3118.py
@@ -112,6 +112,34 @@
 # This table contains format strings as they look on little endian
 # machines.  The test replaces '<' with '>' on big endian machines.
 #
+
+# Platform-specific type codes
+s_bool = {1: '?', 2: 'H', 4: 'L', 8: 'Q'}[sizeof(c_bool)]
+s_short = {2: 'h', 4: 'l', 8: 'q'}[sizeof(c_short)]
+s_ushort = {2: 'H', 4: 'L', 8: 'Q'}[sizeof(c_ushort)]
+s_int = {2: 'h', 4: 'i', 8: 'q'}[sizeof(c_int)]
+s_uint = {2: 'H', 4: 'I', 8: 'Q'}[sizeof(c_uint)]
+s_long = {4: 'l', 8: 'q'}[sizeof(c_long)]
+s_ulong = {4: 'L', 8: 'Q'}[sizeof(c_ulong)]
+s_longlong = "q"
+s_ulonglong = "Q"
+s_float = "f"
+s_double = "d"
+s_longdouble = "g"
+
+# Alias definitions in ctypes/__init__.py
+if c_int is c_long:
+    s_int = s_long
+if c_uint is c_ulong:
+    s_uint = s_ulong
+if c_longlong is c_long:
+    s_longlong = s_long
+if c_ulonglong is c_ulong:
+    s_ulonglong = s_ulong
+if c_longdouble is c_double:
+    s_longdouble = s_double
+
+
 native_types = [
     # type                      format                  shape           calc itemsize
 
@@ -120,52 +148,51 @@
     (c_char,                    "<c",                   None,           c_char),
     (c_byte,                    "<b",                   None,           c_byte),
     (c_ubyte,                   "<B",                   None,           c_ubyte),
-    (c_short,                   "<h",                   None,           c_short),
-    (c_ushort,                  "<H",                   None,           c_ushort),
+    (c_short,                   "<" + s_short,          None,           c_short),
+    (c_ushort,                  "<" + s_ushort,         None,           c_ushort),
 
-    # c_int and c_uint may be aliases to c_long
-    #(c_int,                     "<i",                   None,           c_int),
-    #(c_uint,                    "<I",                   None,           c_uint),
+    (c_int,                     "<" + s_int,            None,           c_int),
+    (c_uint,                    "<" + s_uint,           None,           c_uint),
 
-    (c_long,                    "<l",                   None,           c_long),
-    (c_ulong,                   "<L",                   None,           c_ulong),
+    (c_long,                    "<" + s_long,           None,           c_long),
+    (c_ulong,                   "<" + s_ulong,          None,           c_ulong),
 
-    # c_longlong and c_ulonglong are aliases on 64-bit platforms
-    #(c_longlong,                "<q",                   None,           c_longlong),
-    #(c_ulonglong,               "<Q",                   None,           c_ulonglong),
+    (c_longlong,                "<" + s_longlong,       None,           c_longlong),
+    (c_ulonglong,               "<" + s_ulonglong,      None,           c_ulonglong),
 
     (c_float,                   "<f",                   None,           c_float),
     (c_double,                  "<d",                   None,           c_double),
-    # c_longdouble may be an alias to c_double
 
-    (c_bool,                    "<?",                   None,           c_bool),
+    (c_longdouble,              "<" + s_longdouble,     None,           c_longdouble),
+
+    (c_bool,                    "<" + s_bool,           None,           c_bool),
     (py_object,                 "<O",                   None,           py_object),
 
     ## pointers
 
     (POINTER(c_byte),           "&<b",                  None,           POINTER(c_byte)),
-    (POINTER(POINTER(c_long)),  "&&<l",                 None,           POINTER(POINTER(c_long))),
+    (POINTER(POINTER(c_long)),  "&&<" + s_long,         None,           POINTER(POINTER(c_long))),
 
     ## arrays and pointers
 
     (c_double * 4,              "<d",                   (4,),           c_double),
     (c_float * 4 * 3 * 2,       "<f",                   (2,3,4),        c_float),
-    (POINTER(c_short) * 2,      "&<h",                  (2,),           POINTER(c_short)),
-    (POINTER(c_short) * 2 * 3,  "&<h",                  (3,2,),         POINTER(c_short)),
-    (POINTER(c_short * 2),      "&(2)<h",               None,           POINTER(c_short)),
+    (POINTER(c_short) * 2,      "&<" + s_short,         (2,),           POINTER(c_short)),
+    (POINTER(c_short) * 2 * 3,  "&<" + s_short,         (3,2,),         POINTER(c_short)),
+    (POINTER(c_short * 2),      "&(2)<" + s_short,      None,             POINTER(c_short)),
 
     ## structures and unions
 
-    (Point,                     "T{<l:x:<l:y:}",        None,           Point),
+    (Point,                     "T{<l:x:<l:y:}".replace('l', s_long),  None,  Point),
     # packed structures do not implement the pep
-    (PackedPoint,               "B",                    None,           PackedPoint),
-    (Point2,                    "T{<l:x:<l:y:}",        None,           Point2),
-    (EmptyStruct,               "T{}",                  None,           EmptyStruct),
+    (PackedPoint,               "B",                                   None,  PackedPoint),
+    (Point2,                    "T{<l:x:<l:y:}".replace('l', s_long),  None,  Point2),
+    (EmptyStruct,               "T{}",                                 None,  EmptyStruct),
     # the pep does't support unions
-    (aUnion,                    "B",                    None,           aUnion),
+    (aUnion,                    "B",                                   None,  aUnion),
     # structure with sub-arrays
-    (StructWithArrays,          "T{(2,3)<l:x:(4)T{<l:x:<l:y:}:y:}", None,  StructWithArrays),
-    (StructWithArrays * 3,      "T{(2,3)<l:x:(4)T{<l:x:<l:y:}:y:}", (3,),  StructWithArrays),
+    (StructWithArrays, "T{(2,3)<l:x:(4)T{<l:x:<l:y:}:y:}".replace('l', s_long), None, StructWithArrays),
+    (StructWithArrays * 3, "T{(2,3)<l:x:(4)T{<l:x:<l:y:}:y:}".replace('l', s_long), (3,), StructWithArrays),
 
     ## pointer to incomplete structure
     (Incomplete,                "B",                    None,           Incomplete),
@@ -173,7 +200,7 @@
 
     # 'Complete' is a structure that starts incomplete, but is completed after the
     # pointer type to it has been created.
-    (Complete,                  "T{<l:a:}",             None,           Complete),
+    (Complete,                  "T{<l:a:}".replace('l', s_long), None, Complete),
     # Unfortunately the pointer format string is not fixed...
     (POINTER(Complete),         "&B",                   None,           POINTER(Complete)),
 
@@ -196,10 +223,10 @@
 # and little endian machines.
 #
 endian_types = [
-    (BEPoint,                   "T{>l:x:>l:y:}",        None,           BEPoint),
-    (LEPoint,                   "T{<l:x:<l:y:}",        None,           LEPoint),
-    (POINTER(BEPoint),          "&T{>l:x:>l:y:}",       None,           POINTER(BEPoint)),
-    (POINTER(LEPoint),          "&T{<l:x:<l:y:}",       None,           POINTER(LEPoint)),
+    (BEPoint, "T{>l:x:>l:y:}".replace('l', s_long), None, BEPoint),
+    (LEPoint, "T{<l:x:<l:y:}".replace('l', s_long), None, LEPoint),
+    (POINTER(BEPoint), "&T{>l:x:>l:y:}".replace('l', s_long), None, POINTER(BEPoint)),
+    (POINTER(LEPoint), "&T{<l:x:<l:y:}".replace('l', s_long), None, POINTER(LEPoint)),
     ]
 
 if __name__ == "__main__":
diff --git a/lib-python/2.7/ctypes/test/test_pointers.py b/lib-python/2.7/ctypes/test/test_pointers.py
--- a/lib-python/2.7/ctypes/test/test_pointers.py
+++ b/lib-python/2.7/ctypes/test/test_pointers.py
@@ -210,6 +210,11 @@
         from ctypes import _pointer_type_cache
         del _pointer_type_cache[id(P)]
 
+    def test_abstract(self):
+        from ctypes import _Pointer
+
+        self.assertRaises(TypeError, _Pointer.set_type, 42)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_strings.py b/lib-python/2.7/ctypes/test/test_strings.py
--- a/lib-python/2.7/ctypes/test/test_strings.py
+++ b/lib-python/2.7/ctypes/test/test_strings.py
@@ -63,6 +63,13 @@
 ##        print BUF.from_param(c_char_p("python"))
 ##        print BUF.from_param(BUF(*"pyth"))
 
+    def test_del_segfault(self):
+        BUF = c_char * 4
+        buf = BUF()
+        with self.assertRaises(AttributeError):
+            del buf.raw
+
+
 @need_symbol('c_wchar')
 class WStringArrayTestCase(unittest.TestCase):
     def test(self):
diff --git a/lib-python/2.7/ctypes/test/test_struct_fields.py b/lib-python/2.7/ctypes/test/test_struct_fields.py
--- a/lib-python/2.7/ctypes/test/test_struct_fields.py
+++ b/lib-python/2.7/ctypes/test/test_struct_fields.py
@@ -46,5 +46,29 @@
         Y._fields_ = []
         self.assertRaises(AttributeError, setattr, X, "_fields_", [])
 
+    # __set__ and __get__ should raise a TypeError in case their self
+    # argument is not a ctype instance.
+    def test___set__(self):
+        class MyCStruct(Structure):
+            _fields_ = (("field", c_int),)
+        self.assertRaises(TypeError,
+                          MyCStruct.field.__set__, 'wrong type self', 42)
+
+        class MyCUnion(Union):
+            _fields_ = (("field", c_int),)
+        self.assertRaises(TypeError,
+                          MyCUnion.field.__set__, 'wrong type self', 42)
+
+    def test___get__(self):
+        class MyCStruct(Structure):
+            _fields_ = (("field", c_int),)
+        self.assertRaises(TypeError,
+                          MyCStruct.field.__get__, 'wrong type self', 42)
+
+        class MyCUnion(Union):
+            _fields_ = (("field", c_int),)
+        self.assertRaises(TypeError,
+                          MyCUnion.field.__get__, 'wrong type self', 42)
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_structures.py b/lib-python/2.7/ctypes/test/test_structures.py
--- a/lib-python/2.7/ctypes/test/test_structures.py
+++ b/lib-python/2.7/ctypes/test/test_structures.py
@@ -3,6 +3,7 @@
 from ctypes.test import need_symbol
 from struct import calcsize
 import _testcapi
+import _ctypes_test
 
 class SubclassesTest(unittest.TestCase):
     def test_subclass(self):
@@ -401,6 +402,28 @@
                          (1, 0, 0, 0, 0, 0))
         self.assertRaises(TypeError, lambda: Z(1, 2, 3, 4, 5, 6, 7))
 
+    def test_pass_by_value(self):
+        # This should mirror the structure in Modules/_ctypes/_ctypes_test.c
+        class X(Structure):
+            _fields_ = [
+                ('first', c_ulong),
+                ('second', c_ulong),
+                ('third', c_ulong),
+            ]
+
+        s = X()
+        s.first = 0xdeadbeef
+        s.second = 0xcafebabe
+        s.third = 0x0bad1dea
+        dll = CDLL(_ctypes_test.__file__)
+        func = dll._testfunc_large_struct_update_value
+        func.argtypes = (X,)
+        func.restype = None
+        func(s)
+        self.assertEqual(s.first, 0xdeadbeef)
+        self.assertEqual(s.second, 0xcafebabe)
+        self.assertEqual(s.third, 0x0bad1dea)
+
 class PointerMemberTestCase(unittest.TestCase):
 
     def test(self):
diff --git a/lib-python/2.7/ctypes/test/test_win32.py b/lib-python/2.7/ctypes/test/test_win32.py
--- a/lib-python/2.7/ctypes/test/test_win32.py
+++ b/lib-python/2.7/ctypes/test/test_win32.py
@@ -53,6 +53,24 @@
         windll.user32.GetDesktopWindow()
 
 @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test')
+class ReturnStructSizesTestCase(unittest.TestCase):
+    def test_sizes(self):
+        dll = CDLL(_ctypes_test.__file__)
+        for i in range(1, 11):
+            fields = [ ("f%d" % f, c_char) for f in range(1, i + 1)]
+            class S(Structure):
+                _fields_ = fields
+            f = getattr(dll, "TestSize%d" % i)
+            f.restype = S
+            res = f()
+            for i, f in enumerate(fields):
+                value = getattr(res, f[0])
+                expected = chr(ord('a') + i)
+                self.assertEquals(value, expected)
+
+
+
+ at unittest.skipUnless(sys.platform == "win32", 'Windows-specific test')
 class TestWintypes(unittest.TestCase):
     def test_HWND(self):
         from ctypes import wintypes
diff --git a/lib-python/2.7/curses/ascii.py b/lib-python/2.7/curses/ascii.py
--- a/lib-python/2.7/curses/ascii.py
+++ b/lib-python/2.7/curses/ascii.py
@@ -53,19 +53,19 @@
 
 def isalnum(c): return isalpha(c) or isdigit(c)
 def isalpha(c): return isupper(c) or islower(c)
-def isascii(c): return _ctoi(c) <= 127          # ?
+def isascii(c): return 0 <= _ctoi(c) <= 127          # ?
 def isblank(c): return _ctoi(c) in (9, 32)
-def iscntrl(c): return _ctoi(c) <= 31 or _ctoi(c) == 127
-def isdigit(c): return _ctoi(c) >= 48 and _ctoi(c) <= 57
-def isgraph(c): return _ctoi(c) >= 33 and _ctoi(c) <= 126
-def islower(c): return _ctoi(c) >= 97 and _ctoi(c) <= 122
-def isprint(c): return _ctoi(c) >= 32 and _ctoi(c) <= 126
+def iscntrl(c): return 0 <= _ctoi(c) <= 31 or _ctoi(c) == 127
+def isdigit(c): return 48 <= _ctoi(c) <= 57
+def isgraph(c): return 33 <= _ctoi(c) <= 126
+def islower(c): return 97 <= _ctoi(c) <= 122
+def isprint(c): return 32 <= _ctoi(c) <= 126
 def ispunct(c): return isgraph(c) and not isalnum(c)
 def isspace(c): return _ctoi(c) in (9, 10, 11, 12, 13, 32)
-def isupper(c): return _ctoi(c) >= 65 and _ctoi(c) <= 90
+def isupper(c): return 65 <= _ctoi(c) <= 90
 def isxdigit(c): return isdigit(c) or \
-    (_ctoi(c) >= 65 and _ctoi(c) <= 70) or (_ctoi(c) >= 97 and _ctoi(c) <= 102)
-def isctrl(c): return _ctoi(c) < 32
+    (65 <= _ctoi(c) <= 70) or (97 <= _ctoi(c) <= 102)
+def isctrl(c): return 0 <= _ctoi(c) < 32
 def ismeta(c): return _ctoi(c) > 127
 
 def ascii(c):
diff --git a/lib-python/2.7/curses/has_key.py b/lib-python/2.7/curses/has_key.py
--- a/lib-python/2.7/curses/has_key.py
+++ b/lib-python/2.7/curses/has_key.py
@@ -182,7 +182,7 @@
         L = []
         _curses.initscr()
         for key in _capability_names.keys():
-            system = key in _curses
+            system = _curses.has_key(key)
             python = has_key(key)
             if system != python:
                 L.append( 'Mismatch for key %s, system=%i, Python=%i'
diff --git a/lib-python/2.7/curses/textpad.py b/lib-python/2.7/curses/textpad.py
--- a/lib-python/2.7/curses/textpad.py
+++ b/lib-python/2.7/curses/textpad.py
@@ -43,16 +43,20 @@
     def __init__(self, win, insert_mode=False):
         self.win = win
         self.insert_mode = insert_mode
-        (self.maxy, self.maxx) = win.getmaxyx()
-        self.maxy = self.maxy - 1
-        self.maxx = self.maxx - 1
+        self._update_max_yx()
         self.stripspaces = 1
         self.lastcmd = None
         win.keypad(1)
 
+    def _update_max_yx(self):
+        maxy, maxx = self.win.getmaxyx()
+        self.maxy = maxy - 1
+        self.maxx = maxx - 1
+
     def _end_of_line(self, y):
         """Go to the location of the first blank on the given line,
         returning the index of the last non-blank character."""
+        self._update_max_yx()
         last = self.maxx
         while True:
             if curses.ascii.ascii(self.win.inch(y, last)) != curses.ascii.SP:
@@ -64,8 +68,10 @@
         return last
 
     def _insert_printable_char(self, ch):
+        self._update_max_yx()
         (y, x) = self.win.getyx()
-        if y < self.maxy or x < self.maxx:
+        backyx = None
+        while y < self.maxy or x < self.maxx:
             if self.insert_mode:
                 oldch = self.win.inch()
             # The try-catch ignores the error we trigger from some curses
@@ -75,14 +81,20 @@
                 self.win.addch(ch)
             except curses.error:
                 pass
-            if self.insert_mode:
-                (backy, backx) = self.win.getyx()
-                if curses.ascii.isprint(oldch):
-                    self._insert_printable_char(oldch)
-                    self.win.move(backy, backx)
+            if not self.insert_mode or not curses.ascii.isprint(oldch):
+                break
+            ch = oldch
+            (y, x) = self.win.getyx()
+            # Remember where to put the cursor back since we are in insert_mode
+            if backyx is None:
+                backyx = y, x
+
+        if backyx is not None:
+            self.win.move(*backyx)
 
     def do_command(self, ch):
         "Process a single editing command."
+        self._update_max_yx()
         (y, x) = self.win.getyx()
         self.lastcmd = ch
         if curses.ascii.isprint(ch):
@@ -148,6 +160,7 @@
     def gather(self):
         "Collect and return the contents of the window."
         result = ""
+        self._update_max_yx()
         for y in range(self.maxy+1):
             self.win.move(y, 0)
             stop = self._end_of_line(y)
diff --git a/lib-python/2.7/decimal.py b/lib-python/2.7/decimal.py
--- a/lib-python/2.7/decimal.py
+++ b/lib-python/2.7/decimal.py
@@ -1909,7 +1909,7 @@
         if not other and not self:
             return context._raise_error(InvalidOperation,
                                         'at least one of pow() 1st argument '
-                                        'and 2nd argument must be nonzero ;'
+                                        'and 2nd argument must be nonzero; '
                                         '0**0 is not defined')
 
         # compute sign of result
diff --git a/lib-python/2.7/difflib.py b/lib-python/2.7/difflib.py
--- a/lib-python/2.7/difflib.py
+++ b/lib-python/2.7/difflib.py
@@ -1103,7 +1103,7 @@
 
 import re
 
-def IS_LINE_JUNK(line, pat=re.compile(r"\s*#?\s*$").match):
+def IS_LINE_JUNK(line, pat=re.compile(r"\s*(?:#\s*)?$").match):
     r"""
     Return 1 for ignorable line: iff `line` is blank or contains a single '#'.
 
diff --git a/lib-python/2.7/distutils/archive_util.py b/lib-python/2.7/distutils/archive_util.py
--- a/lib-python/2.7/distutils/archive_util.py
+++ b/lib-python/2.7/distutils/archive_util.py
@@ -162,7 +162,15 @@
             zip = zipfile.ZipFile(zip_filename, "w",
                                   compression=zipfile.ZIP_DEFLATED)
 
+            if base_dir != os.curdir:
+                path = os.path.normpath(os.path.join(base_dir, ''))
+                zip.write(path, path)
+                log.info("adding '%s'", path)
             for dirpath, dirnames, filenames in os.walk(base_dir):
+                for name in dirnames:
+                    path = os.path.normpath(os.path.join(dirpath, name, ''))
+                    zip.write(path, path)
+                    log.info("adding '%s'", path)
                 for name in filenames:
                     path = os.path.normpath(os.path.join(dirpath, name))
                     if os.path.isfile(path):
diff --git a/lib-python/2.7/distutils/ccompiler.py b/lib-python/2.7/distutils/ccompiler.py
--- a/lib-python/2.7/distutils/ccompiler.py
+++ b/lib-python/2.7/distutils/ccompiler.py
@@ -160,7 +160,7 @@
             self.set_executable(key, args[key])
 
     def set_executable(self, key, value):
-        if isinstance(value, str):
+        if isinstance(value, basestring):
             setattr(self, key, split_quoted(value))
         else:
             setattr(self, key, value)
diff --git a/lib-python/2.7/distutils/command/bdist_dumb.py b/lib-python/2.7/distutils/command/bdist_dumb.py
--- a/lib-python/2.7/distutils/command/bdist_dumb.py
+++ b/lib-python/2.7/distutils/command/bdist_dumb.py
@@ -35,7 +35,7 @@
                     ('skip-build', None,
                      "skip rebuilding everything (for testing/debugging)"),
                     ('relative', None,
-                     "build the archive using relative paths"
+                     "build the archive using relative paths "
                      "(default: false)"),
                     ('owner=', 'u',
                      "Owner name used when creating a tar file"
diff --git a/lib-python/2.7/distutils/command/bdist_msi.py b/lib-python/2.7/distutils/command/bdist_msi.py
--- a/lib-python/2.7/distutils/command/bdist_msi.py
+++ b/lib-python/2.7/distutils/command/bdist_msi.py
@@ -99,14 +99,14 @@
                     ('no-target-compile', 'c',
                      "do not compile .py to .pyc on the target system"),
                     ('no-target-optimize', 'o',
-                     "do not compile .py to .pyo (optimized)"
+                     "do not compile .py to .pyo (optimized) "
                      "on the target system"),
                     ('dist-dir=', 'd',
                      "directory to put final built distributions in"),
                     ('skip-build', None,
                      "skip rebuilding everything (for testing/debugging)"),
                     ('install-script=', None,
-                     "basename of installation script to be run after"
+                     "basename of installation script to be run after "
                      "installation or before deinstallation"),
                     ('pre-install-script=', None,
                      "Fully qualified filename of a script to be run before "
diff --git a/lib-python/2.7/distutils/command/bdist_rpm.py b/lib-python/2.7/distutils/command/bdist_rpm.py
--- a/lib-python/2.7/distutils/command/bdist_rpm.py
+++ b/lib-python/2.7/distutils/command/bdist_rpm.py
@@ -63,7 +63,7 @@
          "RPM \"vendor\" (eg. \"Joe Blow <joe at example.com>\") "
          "[default: maintainer or author from setup script]"),
         ('packager=', None,
-         "RPM packager (eg. \"Jane Doe <jane at example.net>\")"
+         "RPM packager (eg. \"Jane Doe <jane at example.net>\") "
          "[default: vendor]"),
         ('doc-files=', None,
          "list of documentation files (space or comma-separated)"),
diff --git a/lib-python/2.7/distutils/command/bdist_wininst.py b/lib-python/2.7/distutils/command/bdist_wininst.py
--- a/lib-python/2.7/distutils/command/bdist_wininst.py
+++ b/lib-python/2.7/distutils/command/bdist_wininst.py
@@ -35,7 +35,7 @@
                     ('no-target-compile', 'c',
                      "do not compile .py to .pyc on the target system"),
                     ('no-target-optimize', 'o',
-                     "do not compile .py to .pyo (optimized)"
+                     "do not compile .py to .pyo (optimized) "
                      "on the target system"),
                     ('dist-dir=', 'd',
                      "directory to put final built distributions in"),
@@ -46,7 +46,7 @@
                     ('skip-build', None,
                      "skip rebuilding everything (for testing/debugging)"),
                     ('install-script=', None,
-                     "basename of installation script to be run after"
+                     "basename of installation script to be run after "
                      "installation or before deinstallation"),
                     ('pre-install-script=', None,
                      "Fully qualified filename of a script to be run before "
diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py
--- a/lib-python/2.7/distutils/command/build_ext.py
+++ b/lib-python/2.7/distutils/command/build_ext.py
@@ -376,7 +376,7 @@
             ext_name, build_info = ext
 
             log.warn(("old-style (ext_name, build_info) tuple found in "
-                      "ext_modules for extension '%s'"
+                      "ext_modules for extension '%s' "
                       "-- please convert to Extension instance" % ext_name))
 
             if not (isinstance(ext_name, str) and
diff --git a/lib-python/2.7/distutils/command/upload.py b/lib-python/2.7/distutils/command/upload.py
--- a/lib-python/2.7/distutils/command/upload.py
+++ b/lib-python/2.7/distutils/command/upload.py
@@ -55,7 +55,9 @@
 
     def run(self):
         if not self.distribution.dist_files:
-            raise DistutilsOptionError("No dist file created in earlier command")
+            msg = ("Must create and upload files in one command "
+                   "(e.g. setup.py sdist upload)")
+            raise DistutilsOptionError(msg)
         for command, pyversion, filename in self.distribution.dist_files:
             self.upload_file(command, pyversion, filename)
 
@@ -155,8 +157,6 @@
                 body.write(fn)
                 body.write("\r\n\r\n")
                 body.write(value)
-                if value and value[-1] == '\r':
-                    body.write('\n')  # write an extra newline (lurve Macs)
         body.write(end_boundary)
         body = body.getvalue()
 
diff --git a/lib-python/2.7/distutils/spawn.py b/lib-python/2.7/distutils/spawn.py
--- a/lib-python/2.7/distutils/spawn.py
+++ b/lib-python/2.7/distutils/spawn.py
@@ -208,7 +208,8 @@
     os.environ['PATH'].  Returns the complete filename or None if not found.
     """
     if path is None:
-        path = os.environ['PATH']
+        path = os.environ.get('PATH', os.defpath)
+
     paths = path.split(os.pathsep)
     base, ext = os.path.splitext(executable)
 
diff --git a/lib-python/2.7/distutils/tests/test_archive_util.py b/lib-python/2.7/distutils/tests/test_archive_util.py
--- a/lib-python/2.7/distutils/tests/test_archive_util.py
+++ b/lib-python/2.7/distutils/tests/test_archive_util.py
@@ -98,7 +98,7 @@
         try:
             names = tar.getnames()
             names.sort()
-            return tuple(names)
+            return names
         finally:
             tar.close()
 
diff --git a/lib-python/2.7/distutils/tests/test_bdist_dumb.py b/lib-python/2.7/distutils/tests/test_bdist_dumb.py
--- a/lib-python/2.7/distutils/tests/test_bdist_dumb.py
+++ b/lib-python/2.7/distutils/tests/test_bdist_dumb.py
@@ -86,7 +86,7 @@
         finally:
             fp.close()
 
-        contents = sorted(os.path.basename(fn) for fn in contents)
+        contents = sorted(filter(None, map(os.path.basename, contents)))
         wanted = ['foo-0.1-py%s.%s.egg-info' % sys.version_info[:2], 'foo.py']
         if not sys.dont_write_bytecode:
             wanted.append('foo.pyc')
diff --git a/lib-python/2.7/distutils/tests/test_bdist_rpm.py b/lib-python/2.7/distutils/tests/test_bdist_rpm.py
--- a/lib-python/2.7/distutils/tests/test_bdist_rpm.py
+++ b/lib-python/2.7/distutils/tests/test_bdist_rpm.py
@@ -99,7 +99,7 @@
     @unittest.skipIf(find_executable('rpmbuild') is None,
                      'the rpmbuild command is not found')
     def test_no_optimize_flag(self):
-        # let's create a package that brakes bdist_rpm
+        # let's create a package that breaks bdist_rpm
         tmp_dir = self.mkdtemp()
         os.environ['HOME'] = tmp_dir   # to confine dir '.rpmdb' creation
         pkg_dir = os.path.join(tmp_dir, 'foo')
diff --git a/lib-python/2.7/distutils/tests/test_build_ext.py b/lib-python/2.7/distutils/tests/test_build_ext.py
--- a/lib-python/2.7/distutils/tests/test_build_ext.py
+++ b/lib-python/2.7/distutils/tests/test_build_ext.py
@@ -20,6 +20,7 @@
 
 class BuildExtTestCase(support.TempdirManager,
                        support.LoggingSilencer,
+                       support.EnvironGuard,
                        unittest.TestCase):
     def setUp(self):
         super(BuildExtTestCase, self).setUp()
diff --git a/lib-python/2.7/distutils/tests/test_ccompiler.py b/lib-python/2.7/distutils/tests/test_ccompiler.py
--- a/lib-python/2.7/distutils/tests/test_ccompiler.py
+++ b/lib-python/2.7/distutils/tests/test_ccompiler.py
@@ -24,6 +24,30 @@
 
 class CCompilerTestCase(support.EnvironGuard, unittest.TestCase):
 
+    def test_set_executables(self):
+        class MyCCompiler(CCompiler):
+            executables = {'compiler': '', 'compiler_cxx': '', 'linker': ''}
+
+        compiler = MyCCompiler()
+
+        # set executable as list
+        compiler.set_executables(compiler=['env', 'OMPI_MPICC=clang', 'mpicc'])
+        self.assertEqual(compiler.compiler, ['env',
+                                             'OMPI_MPICC=clang',
+                                             'mpicc'])
+
+        # set executable as string
+        compiler.set_executables(compiler_cxx='env OMPI_MPICXX=clang++ mpicxx')
+        self.assertEqual(compiler.compiler_cxx, ['env',
+                                                 'OMPI_MPICXX=clang++',
+                                                 'mpicxx'])
+
+        # set executable as unicode string
+        compiler.set_executables(linker=u'env OMPI_MPICXX=clang++ mpiCC')
+        self.assertEqual(compiler.linker, [u'env',
+                                           u'OMPI_MPICXX=clang++',
+                                           u'mpiCC'])
+
     def test_gen_lib_options(self):
         compiler = FakeCompiler()
         libdirs = ['lib1', 'lib2']
diff --git a/lib-python/2.7/distutils/tests/test_check.py b/lib-python/2.7/distutils/tests/test_check.py
--- a/lib-python/2.7/distutils/tests/test_check.py
+++ b/lib-python/2.7/distutils/tests/test_check.py
@@ -8,6 +8,12 @@
 from distutils.tests import support
 from distutils.errors import DistutilsSetupError
 
+try:
+    import pygments
+except ImportError:
+    pygments = None
+
+
 class CheckTestCase(support.LoggingSilencer,
                     support.TempdirManager,
                     unittest.TestCase):
@@ -120,9 +126,15 @@
             pkg_info, dist = self.create_dist(long_description=rest_with_code)
             cmd = check(dist)
             cmd.check_restructuredtext()
-            self.assertEqual(cmd._warnings, 0)
             msgs = cmd._check_rst_data(rest_with_code)
-            self.assertEqual(len(msgs), 0)
+            if pygments is not None:
+                self.assertEqual(len(msgs), 0)
+            else:
+                self.assertEqual(len(msgs), 1)
+                self.assertEqual(
+                    str(msgs[0][1]),
+                    'Cannot analyze code. Pygments package not found.'
+                )
 
     def test_check_all(self):
 
diff --git a/lib-python/2.7/distutils/tests/test_install.py b/lib-python/2.7/distutils/tests/test_install.py
--- a/lib-python/2.7/distutils/tests/test_install.py
+++ b/lib-python/2.7/distutils/tests/test_install.py
@@ -26,6 +26,7 @@
 
 
 class InstallTestCase(support.TempdirManager,
+                      support.EnvironGuard,
                       support.LoggingSilencer,
                       unittest.TestCase):
 
diff --git a/lib-python/2.7/distutils/tests/test_sdist.py b/lib-python/2.7/distutils/tests/test_sdist.py
--- a/lib-python/2.7/distutils/tests/test_sdist.py
+++ b/lib-python/2.7/distutils/tests/test_sdist.py
@@ -130,7 +130,9 @@
             zip_file.close()
 
         # making sure everything has been pruned correctly
-        self.assertEqual(len(content), 4)
+        expected = ['', 'PKG-INFO', 'README', 'setup.py',
+                    'somecode/', 'somecode/__init__.py']
+        self.assertEqual(sorted(content), ['fake-1.0/' + x for x in expected])
 
     @unittest.skipUnless(zlib, "requires zlib")
     def test_make_distribution(self):
@@ -246,7 +248,13 @@
             zip_file.close()
 
         # making sure everything was added
-        self.assertEqual(len(content), 12)
+        expected = ['', 'PKG-INFO', 'README', 'buildout.cfg',
+                    'data/', 'data/data.dt', 'inroot.txt',
+                    'scripts/', 'scripts/script.py', 'setup.py',
+                    'some/', 'some/file.txt', 'some/other_file.txt',
+                    'somecode/', 'somecode/__init__.py', 'somecode/doc.dat',
+                    'somecode/doc.txt']
+        self.assertEqual(sorted(content), ['fake-1.0/' + x for x in expected])
 
         # checking the MANIFEST
         f = open(join(self.tmp_dir, 'MANIFEST'))
diff --git a/lib-python/2.7/distutils/tests/test_spawn.py b/lib-python/2.7/distutils/tests/test_spawn.py
--- a/lib-python/2.7/distutils/tests/test_spawn.py
+++ b/lib-python/2.7/distutils/tests/test_spawn.py
@@ -1,8 +1,11 @@
 """Tests for distutils.spawn."""
+import os
+import stat
+import sys
+import time
 import unittest
-import os
-import time
-from test.test_support import captured_stdout, run_unittest
+from test.support import captured_stdout, run_unittest
+from test import support as test_support
 
 from distutils.spawn import _nt_quote_args
 from distutils.spawn import spawn, find_executable
@@ -53,6 +56,48 @@
         os.chmod(exe, 0777)
         spawn([exe])  # should work without any error
 
+    def test_find_executable(self):
+        with test_support.temp_dir() as tmp_dir:
+            # use TESTFN to get a pseudo-unique filename
+            program_noeext = test_support.TESTFN
+            # Give the temporary program an ".exe" suffix for all.
+            # It's needed on Windows and not harmful on other platforms.
+            program = program_noeext + ".exe"
+
+            filename = os.path.join(tmp_dir, program)
+            with open(filename, "wb"):
+                pass
+            os.chmod(filename, stat.S_IXUSR)
+
+            # test path parameter
+            rv = find_executable(program, path=tmp_dir)
+            self.assertEqual(rv, filename)
+
+            if sys.platform == 'win32':
+                # test without ".exe" extension
+                rv = find_executable(program_noeext, path=tmp_dir)
+                self.assertEqual(rv, filename)
+
+            # test find in the current directory
+            with test_support.change_cwd(tmp_dir):
+                rv = find_executable(program)
+                self.assertEqual(rv, program)
+
+            # test non-existent program
+            dont_exist_program = "dontexist_" + program
+            rv = find_executable(dont_exist_program , path=tmp_dir)
+            self.assertIsNone(rv)
+
+            # test os.defpath: missing PATH environment variable
+            with test_support.EnvironmentVarGuard() as env:
+                from distutils import spawn
+                with test_support.swap_attr(spawn.os, 'defpath', tmp_dir):
+                    env.pop('PATH')
+
+                    rv = find_executable(program)
+                    self.assertEqual(rv, filename)
+
+
 def test_suite():
     return unittest.makeSuite(SpawnTestCase)
 
diff --git a/lib-python/2.7/distutils/tests/test_upload.py b/lib-python/2.7/distutils/tests/test_upload.py
--- a/lib-python/2.7/distutils/tests/test_upload.py
+++ b/lib-python/2.7/distutils/tests/test_upload.py
@@ -128,6 +128,32 @@
         auth = self.last_open.req.headers['Authorization']
         self.assertNotIn('\n', auth)
 
+    # bpo-32304: archives whose last byte was b'\r' were corrupted due to
+    # normalization intended for Mac OS 9.
+    def test_upload_correct_cr(self):
+        # content that ends with \r should not be modified.
+        tmp = self.mkdtemp()

From pypy.commits at gmail.com  Mon May 20 07:30:44 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 20 May 2019 04:30:44 -0700 (PDT)
Subject: [pypy-commit] pypy stdlib-2.7.16: add _ssl.HAS_TLSv1_3
Message-ID: <5ce28fe4.1c69fb81.28242.c9ef@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: stdlib-2.7.16
Changeset: r96644:c59a0f5fdb0d
Date: 2019-05-19 23:14 +0300
http://bitbucket.org/pypy/pypy/changeset/c59a0f5fdb0d/

Log:	add _ssl.HAS_TLSv1_3

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -103,7 +103,10 @@
 constants["OPENSSL_VERSION_INFO"] = version_info
 constants["_OPENSSL_API_VERSION"] = version_info
 constants["OPENSSL_VERSION"] = SSLEAY_VERSION
-
+if OPENSSL_NO_TLS1_3:
+    constants["HAS_TLSv1_3"] = 0
+else:
+    constants["HAS_TLSv1_3"] = TLS1_3_VERSION
 
 def ssl_error(space, msg, errno=0, w_errtype=None, errcode=0):
     reason_str = None
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -42,6 +42,7 @@
         assert 'openssl' in lower_version or "libressl" in lower_version
 
         assert isinstance(_ssl.ALERT_DESCRIPTION_ACCESS_DENIED, int)
+        assert isinstance(_ssl.HAS_TLSv1_3, int)
 
     def test_RAND_add(self):
         import _ssl
diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -132,6 +132,8 @@
         SSL_OP_NO_TLSv1_1 = rffi_platform.ConstantInteger("SSL_OP_NO_TLSv1_1")
         SSL_OP_NO_TLSv1_2 = rffi_platform.ConstantInteger("SSL_OP_NO_TLSv1_2")
     OPENSSL_NO_TLSEXT = rffi_platform.Defined("OPENSSL_NO_TLSEXT")
+    OPENSSL_NO_TLS1_3 = rffi_platform.Defined("OPENSSL_NO_TLS1_3")
+    TLS1_3_VERSION = rffi_platform.Defined("TLS1_3_VERSION")
     SSL_OP_CIPHER_SERVER_PREFERENCE = rffi_platform.ConstantInteger(
         "SSL_OP_CIPHER_SERVER_PREFERENCE")
     SSL_OP_SINGLE_DH_USE = rffi_platform.ConstantInteger(

From pypy.commits at gmail.com  Mon May 20 09:06:06 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 20 May 2019 06:06:06 -0700 (PDT)
Subject: [pypy-commit] pypy stdlib-2.7.16: register new tests
Message-ID: <5ce2a63e.1c69fb81.4fc3c.234f@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: stdlib-2.7.16
Changeset: r96645:51b61fa2015d
Date: 2019-05-20 16:05 +0300
http://bitbucket.org/pypy/pypy/changeset/51b61fa2015d/

Log:	register new tests

diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -115,6 +115,7 @@
     RegrTest('test_augassign.py', core=True),
     RegrTest('test_base64.py', usemodules='struct'),
     RegrTest('test_bastion.py'),
+    RegrTest('test_bdb.py'),
     RegrTest('test_bigaddrspace.py'),
     RegrTest('test_bigmem.py'),
     RegrTest('test_binascii.py', usemodules='binascii'),
@@ -269,6 +270,7 @@
     RegrTest('test_imghdr.py'),
     RegrTest('test_imp.py', core=True, usemodules='thread'),
     RegrTest('test_import.py', core=True),
+    RegrTest('test_import_magic.py'),
     RegrTest('test_importhooks.py', core=True),
     RegrTest('test_importlib.py'),
     RegrTest('test_index.py'),
@@ -369,6 +371,7 @@
     RegrTest('test_random.py'),
     RegrTest('test_re.py', core=True),
     RegrTest('test_readline.py'),
+    RegrTest('test_regrtest.py'),
     RegrTest('test_repr.py', core=True),
     RegrTest('test_resource.py'),
     RegrTest('test_rfc822.py'),
@@ -419,6 +422,7 @@
     RegrTest('test_sunau.py'),
     RegrTest('test_sunaudiodev.py'),
     RegrTest('test_sundry.py'),
+    RegrTest('test_test_support.py'),
     RegrTest('test_symtable.py', skip="implementation detail"),
     RegrTest('test_syntax.py', core=True),
     RegrTest('test_sys.py', core=True, usemodules='struct'),

From pypy.commits at gmail.com  Mon May 20 12:35:53 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 20 May 2019 09:35:53 -0700 (PDT)
Subject: [pypy-commit] pypy default: update certificates per cpython issue
 36816 PR 13199
Message-ID: <5ce2d769.1c69fb81.e2b31.f8c5@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r96646:4560d56a123a
Date: 2019-05-20 19:34 +0300
http://bitbucket.org/pypy/pypy/changeset/4560d56a123a/

Log:	update certificates per cpython issue 36816 PR 13199

diff --git a/lib-python/2.7/test/capath/efa5f9c3.0 b/lib-python/2.7/test/capath/efa5f9c3.0
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/capath/efa5f9c3.0
@@ -0,0 +1,34 @@
+-----BEGIN CERTIFICATE-----
+MIIF9zCCA9+gAwIBAgIUH98b4Fw/DyugC9cV7VK7ZODzHsIwDQYJKoZIhvcNAQEL
+BQAwgYoxCzAJBgNVBAYTAlhZMRcwFQYDVQQIDA5DYXN0bGUgQW50aHJheDEYMBYG
+A1UEBwwPQXJndW1lbnQgQ2xpbmljMSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUg
+Rm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0aG9udGVzdC5uZXQw
+HhcNMTkwNTA4MDEwMjQzWhcNMjcwNzI0MDEwMjQzWjCBijELMAkGA1UEBhMCWFkx
+FzAVBgNVBAgMDkNhc3RsZSBBbnRocmF4MRgwFgYDVQQHDA9Bcmd1bWVudCBDbGlu
+aWMxIzAhBgNVBAoMGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMSMwIQYDVQQD
+DBpzZWxmLXNpZ25lZC5weXRob250ZXN0Lm5ldDCCAiIwDQYJKoZIhvcNAQEBBQAD
+ggIPADCCAgoCggIBAMKdJlyCThkahwoBb7pl5q64Pe9Fn5jrIvzsveHTc97TpjV2
+RLfICnXKrltPk/ohkVl6K5SUZQZwMVzFubkyxE0nZPHYHlpiKWQxbsYVkYv01rix
+IFdLvaxxbGYke2jwQao31s4o61AdlsfK1SdpHQUynBBMssqI3SB4XPmcA7e+wEEx
+jxjVish4ixA1vuIZOx8yibu+CFCf/geEjoBMF3QPdzULzlrCSw8k/45iZCSoNbvK
+DoL4TVV07PHOxpheDh8ZQmepGvU6pVqhb9m4lgmV0OGWHgozd5Ur9CbTVDmxIEz3
+TSoRtNJK7qtyZdGNqwjksQxgZTjM/d/Lm/BJG99AiOmYOjsl9gbQMZgvQmMAtUsI
+aMJnQuZ6R+KEpW/TR5qSKLWZSG45z/op+tzI2m+cE6HwTRVAWbcuJxcAA55MZjqU
+OOOu3BBYMjS5nf2sQ9uoXsVBFH7i0mQqoW1SLzr9opI8KsWwFxQmO2vBxWYaN+lH
+OmwBZBwyODIsmI1YGXmTp09NxRYz3Qe5GCgFzYowpMrcxUC24iduIdMwwhRM7rKg
+7GtIWMSrFfuI1XCLRmSlhDbhNN6fVg2f8Bo9PdH9ihiIyxSrc+FOUasUYCCJvlSZ
+8hFUlLvcmrZlWuazohm0lsXuMK1JflmQr/DA/uXxP9xzFfRy+RU3jDyxJbRHAgMB
+AAGjUzBRMB0GA1UdDgQWBBSQJyxiPMRK01i+0BsV9zUwDiBaHzAfBgNVHSMEGDAW
+gBSQJyxiPMRK01i+0BsV9zUwDiBaHzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3
+DQEBCwUAA4ICAQCR+7a7N/m+WLkxPPIA/CB4MOr2Uf8ixTv435Nyv6rXOun0+lTP
+ExSZ0uYQ+L0WylItI3cQHULldDueD+s8TGzxf5woaLKf6tqyr0NYhKs+UeNEzDnN
+9PHQIhX0SZw3XyXGUgPNBfRCg2ZDdtMMdOU4XlQN/IN/9hbYTrueyY7eXq9hmtI9
+1srftAMqr9SR1JP7aHI6DVgrEsZVMTDnfT8WmLSGLlY1HmGfdEn1Ip5sbo9uSkiH
+AEPgPfjYIvR5LqTOMn4KsrlZyBbFIDh9Sl99M1kZzgH6zUGVLCDg1y6Cms69fx/e
+W1HoIeVkY4b4TY7Bk7JsqyNhIuqu7ARaxkdaZWhYaA2YyknwANdFfNpfH+elCLIk
+BUt5S3f4i7DaUePTvKukCZiCq4Oyln7RcOn5If73wCeLB/ZM9Ei1HforyLWP1CN8
+XLfpHaoeoPSWIveI0XHUl65LsPN2UbMbul/F23hwl+h8+BLmyAS680Yhn4zEN6Ku
+B7Po90HoFa1Du3bmx4jsN73UkT/dwMTi6K072FbipnC1904oGlWmLwvAHvrtxxmL
+Pl3pvEaZIu8wa/PNF6Y7J7VIewikIJq6Ta6FrWeFfzMWOj2qA1ZZi6fUaDSNYvuV
+J5quYKCc/O+I/yDDf8wyBbZ/gvUXzUHTMYGG+bFrn1p7XDbYYeEJ6R/xEg==
+-----END CERTIFICATE-----
diff --git a/lib-python/2.7/test/selfsigned_pythontestdotnet.pem b/lib-python/2.7/test/selfsigned_pythontestdotnet.pem
--- a/lib-python/2.7/test/selfsigned_pythontestdotnet.pem
+++ b/lib-python/2.7/test/selfsigned_pythontestdotnet.pem
@@ -1,16 +1,34 @@
 -----BEGIN CERTIFICATE-----
-MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV
-BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u
-IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv
-bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG
-A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo
-b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0
-aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ
-Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm
-Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv
-EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl
-bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN
-AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h
-TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515
-C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM=
+MIIF9zCCA9+gAwIBAgIUH98b4Fw/DyugC9cV7VK7ZODzHsIwDQYJKoZIhvcNAQEL
+BQAwgYoxCzAJBgNVBAYTAlhZMRcwFQYDVQQIDA5DYXN0bGUgQW50aHJheDEYMBYG
+A1UEBwwPQXJndW1lbnQgQ2xpbmljMSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUg
+Rm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0aG9udGVzdC5uZXQw
+HhcNMTkwNTA4MDEwMjQzWhcNMjcwNzI0MDEwMjQzWjCBijELMAkGA1UEBhMCWFkx
+FzAVBgNVBAgMDkNhc3RsZSBBbnRocmF4MRgwFgYDVQQHDA9Bcmd1bWVudCBDbGlu
+aWMxIzAhBgNVBAoMGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMSMwIQYDVQQD
+DBpzZWxmLXNpZ25lZC5weXRob250ZXN0Lm5ldDCCAiIwDQYJKoZIhvcNAQEBBQAD
+ggIPADCCAgoCggIBAMKdJlyCThkahwoBb7pl5q64Pe9Fn5jrIvzsveHTc97TpjV2
+RLfICnXKrltPk/ohkVl6K5SUZQZwMVzFubkyxE0nZPHYHlpiKWQxbsYVkYv01rix
+IFdLvaxxbGYke2jwQao31s4o61AdlsfK1SdpHQUynBBMssqI3SB4XPmcA7e+wEEx
+jxjVish4ixA1vuIZOx8yibu+CFCf/geEjoBMF3QPdzULzlrCSw8k/45iZCSoNbvK
+DoL4TVV07PHOxpheDh8ZQmepGvU6pVqhb9m4lgmV0OGWHgozd5Ur9CbTVDmxIEz3
+TSoRtNJK7qtyZdGNqwjksQxgZTjM/d/Lm/BJG99AiOmYOjsl9gbQMZgvQmMAtUsI
+aMJnQuZ6R+KEpW/TR5qSKLWZSG45z/op+tzI2m+cE6HwTRVAWbcuJxcAA55MZjqU
+OOOu3BBYMjS5nf2sQ9uoXsVBFH7i0mQqoW1SLzr9opI8KsWwFxQmO2vBxWYaN+lH
+OmwBZBwyODIsmI1YGXmTp09NxRYz3Qe5GCgFzYowpMrcxUC24iduIdMwwhRM7rKg
+7GtIWMSrFfuI1XCLRmSlhDbhNN6fVg2f8Bo9PdH9ihiIyxSrc+FOUasUYCCJvlSZ
+8hFUlLvcmrZlWuazohm0lsXuMK1JflmQr/DA/uXxP9xzFfRy+RU3jDyxJbRHAgMB
+AAGjUzBRMB0GA1UdDgQWBBSQJyxiPMRK01i+0BsV9zUwDiBaHzAfBgNVHSMEGDAW
+gBSQJyxiPMRK01i+0BsV9zUwDiBaHzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3
+DQEBCwUAA4ICAQCR+7a7N/m+WLkxPPIA/CB4MOr2Uf8ixTv435Nyv6rXOun0+lTP
+ExSZ0uYQ+L0WylItI3cQHULldDueD+s8TGzxf5woaLKf6tqyr0NYhKs+UeNEzDnN
+9PHQIhX0SZw3XyXGUgPNBfRCg2ZDdtMMdOU4XlQN/IN/9hbYTrueyY7eXq9hmtI9
+1srftAMqr9SR1JP7aHI6DVgrEsZVMTDnfT8WmLSGLlY1HmGfdEn1Ip5sbo9uSkiH
+AEPgPfjYIvR5LqTOMn4KsrlZyBbFIDh9Sl99M1kZzgH6zUGVLCDg1y6Cms69fx/e
+W1HoIeVkY4b4TY7Bk7JsqyNhIuqu7ARaxkdaZWhYaA2YyknwANdFfNpfH+elCLIk
+BUt5S3f4i7DaUePTvKukCZiCq4Oyln7RcOn5If73wCeLB/ZM9Ei1HforyLWP1CN8
+XLfpHaoeoPSWIveI0XHUl65LsPN2UbMbul/F23hwl+h8+BLmyAS680Yhn4zEN6Ku
+B7Po90HoFa1Du3bmx4jsN73UkT/dwMTi6K072FbipnC1904oGlWmLwvAHvrtxxmL
+Pl3pvEaZIu8wa/PNF6Y7J7VIewikIJq6Ta6FrWeFfzMWOj2qA1ZZi6fUaDSNYvuV
+J5quYKCc/O+I/yDDf8wyBbZ/gvUXzUHTMYGG+bFrn1p7XDbYYeEJ6R/xEg==
 -----END CERTIFICATE-----

From pypy.commits at gmail.com  Mon May 20 15:22:08 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 20 May 2019 12:22:08 -0700 (PDT)
Subject: [pypy-commit] pypy cffi-libs: backport package.py,
 build_cffi_imports from py3.6
Message-ID: <5ce2fe60.1c69fb81.911b6.fe85@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cffi-libs
Changeset: r96648:8ae1951978c7
Date: 2019-05-20 22:21 +0300
http://bitbucket.org/pypy/pypy/changeset/8ae1951978c7/

Log:	backport package.py, build_cffi_imports from py3.6

diff --git a/pypy/tool/build_cffi_imports.py b/pypy/tool/build_cffi_imports.py
--- a/pypy/tool/build_cffi_imports.py
+++ b/pypy/tool/build_cffi_imports.py
@@ -1,11 +1,13 @@
 from __future__ import print_function
-import sys, shutil, os
+import sys, shutil, os, tempfile, hashlib
+from os.path import join
 
 class MissingDependenciesError(Exception):
     pass
 
 
 cffi_build_scripts = {
+    "_ssl": "_ssl_build.py",
     "sqlite3": "_sqlite3_build.py",
     "audioop": "_audioop_build.py",
     "tk": "_tkinter/tklib_build.py",
@@ -17,10 +19,125 @@
     "xx": None,    # for testing: 'None' should be completely ignored
     }
 
-def create_cffi_import_libraries(pypy_c, options, basedir):
+# for distribution, we may want to fetch dependencies not provided by
+# the OS, such as a recent openssl/libressl.
+cffi_dependencies = {
+    '_ssl': ('http://ftp.openbsd.org/pub/OpenBSD/LibreSSL/libressl-2.6.2.tar.gz',
+            'b029d2492b72a9ba5b5fcd9f3d602c9fd0baa087912f2aaecc28f52f567ec478',
+            ['--without-openssldir']),
+    '_gdbm': ('http://ftp.gnu.org/gnu/gdbm/gdbm-1.13.tar.gz',
+              '9d252cbd7d793f7b12bcceaddda98d257c14f4d1890d851c386c37207000a253',
+              ['--without-readline']),
+}
+
+
+def _unpack_tarfile(filename, extract_dir):
+    """Unpack tar/tar.gz/tar.bz2/tar.xz `filename` to `extract_dir`
+    """
+    import tarfile  # late import for breaking circular dependency
+    try:
+        tarobj = tarfile.open(filename)
+    except tarfile.TarError:
+        raise ReadError(
+            "%s is not a compressed or uncompressed tar file" % filename)
+    try:
+        tarobj.extractall(extract_dir)
+    finally:
+        tarobj.close()
+
+def _sha256(filename):
+    dgst = hashlib.sha256()
+
+    with open(filename, 'rb') as fp:
+        dgst.update(fp.read())
+    return dgst.hexdigest()
+
+
+def _build_dependency(name, destdir, patches=[]):
+    import multiprocessing
+    import shutil
+    import subprocess
+
     from rpython.tool.runsubprocess import run_subprocess
 
-    shutil.rmtree(str(basedir.join('lib_pypy', '__pycache__')),
+    try:
+        from urllib.request import urlretrieve
+    except ImportError:
+        from urllib import urlretrieve
+
+    try:
+        url, dgst, args = cffi_dependencies[name]
+    except KeyError:
+        return 0, None, None
+
+    archive_dir = os.path.join(tempfile.gettempdir(), 'pypy-archives')
+
+    if not os.path.isdir(archive_dir):
+        os.makedirs(archive_dir)
+
+    archive = os.path.join(archive_dir, url.rsplit('/', 1)[-1])
+
+    # next, fetch the archive to disk, if needed
+    if not os.path.exists(archive) or _sha256(archive) != dgst:
+        print('fetching archive', url, file=sys.stderr)
+        urlretrieve(url, archive)
+
+    # extract the archive into our destination directory
+    print('unpacking archive', archive, file=sys.stderr)
+    _unpack_tarfile(archive, destdir)
+
+    sources = os.path.join(
+        destdir,
+        os.path.basename(archive)[:-7],
+    )
+
+    # apply any patches
+    if patches:
+        for patch in patches:
+            print('applying patch', patch, file=sys.stderr)
+            status, stdout, stderr = run_subprocess(
+                '/usr/bin/patch', ['-p1', '-i', patch], cwd=sources,
+            )
+
+            if status != 0:
+                return status, stdout, stderr
+
+    print('configuring', sources, file=sys.stderr)
+
+    # configure & build it
+    status, stdout, stderr = run_subprocess(
+        './configure',
+        [
+            '--prefix=/usr',
+            '--disable-shared',
+            '--enable-silent-rules',
+            '--disable-dependency-tracking',
+        ] + args,
+        cwd=sources,
+    )
+
+    if status != 0:
+        return status, stdout, stderr
+
+    print('building', sources, file=sys.stderr)
+
+    status, stdout, stderr = run_subprocess(
+        'make',
+        [
+            '-s', '-j' + str(multiprocessing.cpu_count()),
+            'install', 'DESTDIR={}/'.format(destdir),
+        ],
+        cwd=sources,
+    )
+
+    return status, stdout, stderr
+
+
+def create_cffi_import_libraries(pypy_c, options, basedir, only=None,
+                                 embed_dependencies=False):
+    from rpython.tool.runsubprocess import run_subprocess
+
+    shutil.rmtree(str(join(basedir,'lib_pypy','__pycache__')),
                   ignore_errors=True)
     # be sure pip, setuptools are installed in a fresh pypy
     # allows proper functioning of cffi on win32 with newer vc compilers
@@ -29,24 +146,72 @@
     if status  != 0:
         status, stdout, stderr = run_subprocess(str(pypy_c), ['-m', 'ensurepip'])
     failures = []
-    env = os.environ.copy()
-    if sys.platform == 'win32':
-        env['INCLUDE'] = r'..\externals\include;' + env.get('INCLUDE', '')
-        env['LIB'] = r'..\externals\lib;' + env.get('LIB', '')
-        env['PATH'] = r'..\externals\bin;' + env.get('PATH', '')
-    for key, module in sorted(cffi_build_scripts.items()):
+
+    for key, module in cffi_build_scripts.items():
+        if only and key not in only:
+            print("* SKIPPING", key, '(not specified in --only)')
+            continue
         if module is None or getattr(options, 'no_' + key, False):
             continue
+        # the key is the module name, has it already been built?
+        status, stdout, stderr = run_subprocess(str(pypy_c), ['-c', 'import %s' % key])
+        if status  == 0:
+            print('*', ' %s already built' % key, file=sys.stderr)
+            continue
+        
         if module.endswith('.py'):
             args = [module]
-            cwd = str(basedir.join('lib_pypy'))
+            cwd = str(join(basedir,'lib_pypy'))
         else:
             args = ['-c', 'import ' + module]
             cwd = None
+        env = os.environ.copy()
+
         print('*', ' '.join(args), file=sys.stderr)
+        if embed_dependencies:
+            curdir = os.path.abspath(os.path.dirname(__file__))
+            destdir = os.path.join(curdir, 'dest')
+
+            shutil.rmtree(destdir, ignore_errors=True)
+            os.makedirs(destdir)
+
+            if key == '_ssl' and sys.platform == 'darwin':
+                # this patch is loosely inspired by an Apple and adds
+                # a fallback to the OS X roots when none are available
+                patches = [
+                    os.path.join(curdir,
+                                 '../../lib_pypy/_cffi_ssl/osx-roots.diff'),
+                ]
+            else:
+                patches = []
+
+            status, stdout, stderr = _build_dependency(key, destdir,
+                                                       patches=patches)
+
+            if status != 0:
+                failures.append((key, module))
+                print("stdout:")
+                print(stdout.decode('utf-8'))
+                print("stderr:")
+                print(stderr.decode('utf-8'))
+                continue
+
+            env['CPPFLAGS'] = \
+                '-I{}/usr/include {}'.format(destdir, env.get('CPPFLAGS', ''))
+            env['LDFLAGS'] = \
+                '-L{}/usr/lib {}'.format(destdir, env.get('LDFLAGS', ''))
+
+            if key == '_ssl' and sys.platform == 'darwin':
+                # needed for our roots patch
+                env['LDFLAGS'] += ' -framework CoreFoundation -framework Security'
+        elif sys.platform == 'win32':
+            env['INCLUDE'] = r'..\externals\include;' + env.get('INCLUDE', '')
+            env['LIB'] = r'..\externals\lib;' + env.get('LIB', '')
+            env['PATH'] = r'..\externals\bin;' + env.get('PATH', '')
+
         try:
             status, stdout, stderr = run_subprocess(str(pypy_c), args,
-                                                     cwd=cwd, env=env)
+                                                    cwd=cwd, env=env)
             if status != 0:
                 print(stdout, stderr, file=sys.stderr)
                 failures.append((key, module))
@@ -56,6 +221,7 @@
     return failures
 
 if __name__ == '__main__':
+    import argparse
     if '__pypy__' not in sys.builtin_module_names:
         print('Call with a pypy interpreter', file=sys.stderr)
         sys.exit(1)
@@ -64,21 +230,35 @@
     base_dir = os.path.dirname(os.path.dirname(tool_dir))
     sys.path.insert(0, base_dir)
 
-    import py
-
     class Options(object):
         pass
 
-    exename = py.path.local(sys.executable) 
+    parser = argparse.ArgumentParser(description='Build all cffi backends in lib_pypy')
+    parser.add_argument('--exefile', dest='exefile', default=sys.executable,
+                        help='instead of executing sys.executable' \
+                             ' you can specify an alternative pypy vm here')
+    parser.add_argument('--only', dest='only', default=None,
+                        help='Only build the modules delimited by a colon. E.g. _ssl,sqlite')
+    parser.add_argument('--embed-dependencies', dest='embed_dependencies', action='store_true',
+        help='embed dependencies for distribution')
+    args = parser.parse_args()
+
+    exename = join(os.getcwd(), args.exefile)
     basedir = exename
-    while not basedir.join('include').exists():
-        _basedir = basedir.dirpath()
+
+    while not os.path.exists(join(basedir,'include')):
+        _basedir = os.path.dirname(basedir)
         if _basedir == basedir:
             raise ValueError('interpreter %s not inside pypy repo', 
                                  str(exename))
         basedir = _basedir
     options = Options()
-    failures = create_cffi_import_libraries(exename, options, basedir)
+    if args.only is None:
+        only = None
+    else:
+        only = set(args.only.split(','))
+    failures = create_cffi_import_libraries(exename, options, basedir, only=only,
+                                            embed_dependencies=args.embed_dependencies)
     if len(failures) > 0:
         print('*** failed to build the CFFI modules %r' % (
             [f[1] for f in failures],), file=sys.stderr)
@@ -97,7 +277,7 @@
         for k in cffi_build_scripts:
             setattr(options, 'no_' + k, True)
         must_fail = '_missing_build_script.py'
-        assert not os.path.exists(str(basedir.join('lib_pypy').join(must_fail)))
+        assert not os.path.exists(str(join(join(basedir,'lib_pypy'),must_fail)))
         cffi_build_scripts['should_fail'] = must_fail
-        failures = create_cffi_import_libraries(exename, options, basedir)
+        failures = create_cffi_import_libraries(exename, options, basedir, only=only)
         assert len(failures) == 1
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -83,7 +83,11 @@
     if not _fake and not pypy_runs(pypy_c):
         raise OSError("Running %r failed!" % (str(pypy_c),))
     if not options.no_cffi:
-        failures = create_cffi_import_libraries(pypy_c, options, basedir)
+        failures = create_cffi_import_libraries(
+            str(pypy_c), options, str(basedir),
+            embed_dependencies=options.embed_dependencies,
+        )
+
         for key, module in failures:
             print >>sys.stderr, """!!!!!!!!!!\nBuilding {0} bindings failed.
                 You can either install development headers package,
@@ -262,11 +266,16 @@
     return retval, builddir # for tests
 
 def package(*args, **kwds):
-    try:
-        import argparse
-    except ImportError:
-        import imp
-        argparse = imp.load_source('argparse', 'lib-python/2.7/argparse.py')
+    import argparse
+
+    class NegateAction(argparse.Action):
+        def __init__(self, option_strings, dest, nargs=0, **kwargs):
+            super(NegateAction, self).__init__(option_strings, dest, nargs,
+                                               **kwargs)
+
+        def __call__(self, parser, ns, values, option):
+            setattr(ns, self.dest, option[2:4] != 'no')
+
     if sys.platform == 'win32':
         pypy_exe = 'pypy.exe'
     else:
@@ -288,7 +297,7 @@
     parser.add_argument('--no-keep-debug', dest='keep_debug',
                         action='store_false', help='do not keep debug symbols')
     parser.add_argument('--rename_pypy_c', dest='pypy_c', type=str, default=pypy_exe,
-        help='target executable name, defaults to "pypy"')
+        help='target executable name, defaults to "%s"' % pypy_exe)
     parser.add_argument('--archive-name', dest='name', type=str, default='',
         help='pypy-VER-PLATFORM')
     parser.add_argument('--builddir', type=str, default='',
@@ -296,13 +305,21 @@
     parser.add_argument('--targetdir', type=str, default='',
         help='destination dir for archive')
     parser.add_argument('--override_pypy_c', type=str, default='',
-        help='use as pypy exe instead of pypy/goal/pypy-c')
+        help='use as pypy3 exe instead of pypy/goal/pypy3-c')
+    parser.add_argument('--embedded-dependencies', '--no-embedded-dependencies',
+                        dest='embed_dependencies',
+                        action=NegateAction,
+                        default=(sys.platform == 'darwin'),
+                        help='whether to embed dependencies for distribution '
+                        '(default on OS X)')
     options = parser.parse_args(args)
 
     if os.environ.has_key("PYPY_PACKAGE_NOKEEPDEBUG"):
         options.keep_debug = False
     if os.environ.has_key("PYPY_PACKAGE_WITHOUTTK"):
         options.no_tk = True
+    if os.environ.has_key("PYPY_EMBED_DEPENDENCIES"):
+        options.embed_dependencies = True
     if not options.builddir:
         # The import actually creates the udir directory
         from rpython.tool.udir import udir

From pypy.commits at gmail.com  Mon May 20 16:17:04 2019
From: pypy.commits at gmail.com (andrewjlawrence)
Date: Mon, 20 May 2019 13:17:04 -0700 (PDT)
Subject: [pypy-commit] pypy winmultiprocessing: Corrected parameters to
 RSocket call.
Message-ID: <5ce30b40.1c69fb81.82a96.3e6e@mx.google.com>

Author: andrewjlawrence
Branch: winmultiprocessing
Changeset: r96649:884c7e6b4d3a
Date: 2019-05-20 21:15 +0100
http://bitbucket.org/pypy/pypy/changeset/884c7e6b4d3a/

Log:	Corrected parameters to RSocket call.

diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -207,7 +207,7 @@
                     from rpython.rlib.rsocket import _c
                     # it is possible to pass some bytes representing a socket
                     # in the file descriptor object on winodws
-                    fdobj = space.text_w(w_fdobj)
+                    fdobj = space.bytes_w(w_fdobj)
                     info_charptr = rffi.str2charp(fdobj)
                     try:
                         info_ptr = rffi.cast(lltype.Ptr(_c.WSAPROTOCOL_INFOW), info_charptr)
@@ -215,7 +215,7 @@
                         _c.FROM_PROTOCOL_INFO, info_ptr, 0, _c.WSA_FLAG_OVERLAPPED)
                         if fd == rsocket.INVALID_SOCKET:
                             raise converted_error(space, rsocket.last_error())
-                        sock = RSocket(family, type, proto, fd)
+                        sock = RSocket(info_ptr.c_iAddressFamily, info_ptr.c_iSocketType, info_ptr.c_iProtocol, fd)
                     finally:
                         lltype.free(info_charptr, flavor='raw')
                 else:

From pypy.commits at gmail.com  Tue May 21 00:53:42 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 20 May 2019 21:53:42 -0700 (PDT)
Subject: [pypy-commit] pypy default: avoid void* arithmetic by changing vp
 += val to vp = (char*)vp + v, fix win32?
Message-ID: <5ce38456.1c69fb81.1123b.5caa@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r96650:1939f3b020f8
Date: 2019-05-21 06:53 +0300
http://bitbucket.org/pypy/pypy/changeset/1939f3b020f8/

Log:	avoid void* arithmetic by changing vp += val to vp = (char*)vp + v,
	fix win32?

diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py
--- a/rpython/translator/c/gc.py
+++ b/rpython/translator/c/gc.py
@@ -461,10 +461,12 @@
         raise Exception("gc_pop_roots should be removed by postprocess_graph")
 
     def OP_GC_ENTER_ROOTS_FRAME(self, funcgen, op):
-        return '(%s) += sizeof(pypy_ss_t);' % (funcgen.gcpol_ss,)
+        # avoid arithmatic on void*
+        return '({0}) = (char*)({0}) + sizeof(pypy_ss_t);'.format(funcgen.gcpol_ss,)
 
     def OP_GC_LEAVE_ROOTS_FRAME(self, funcgen, op):
-        return '(%s) -= sizeof(pypy_ss_t);' % (funcgen.gcpol_ss,)
+        # avoid arithmatic on void*
+        return '({0}) = (char*)({0}) - sizeof(pypy_ss_t);'.format(funcgen.gcpol_ss,)
 
     def OP_GC_SAVE_ROOT(self, funcgen, op):
         num = op.args[0].value

From pypy.commits at gmail.com  Tue May 21 00:53:44 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 20 May 2019 21:53:44 -0700 (PDT)
Subject: [pypy-commit] pypy cffi-libs: fix mro of SSLError
Message-ID: <5ce38458.1c69fb81.2efd2.0736@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cffi-libs
Changeset: r96651:7bd108e790be
Date: 2019-05-21 07:53 +0300
http://bitbucket.org/pypy/pypy/changeset/7bd108e790be/

Log:	fix mro of SSLError

diff --git a/lib_pypy/_cffi_ssl/_stdssl/error.py b/lib_pypy/_cffi_ssl/_stdssl/error.py
--- a/lib_pypy/_cffi_ssl/_stdssl/error.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/error.py
@@ -1,6 +1,7 @@
 import sys
 import os
 import traceback
+import socket
 from _pypy_openssl import ffi
 from _pypy_openssl import lib
 
@@ -20,7 +21,7 @@
 SSL_ERROR_NO_SOCKET = 9 # socket has been GC'd
 SSL_ERROR_INVALID_ERROR_CODE = 10
 
-class SSLError(OSError):
+class SSLError(socket.error):
     """ An error occurred in the SSL implementation. """
     def __str__(self):
         if self.strerror and isinstance(self.strerror, str):

From pypy.commits at gmail.com  Tue May 21 01:36:23 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 20 May 2019 22:36:23 -0700 (PDT)
Subject: [pypy-commit] pypy cffi-libs: nicer cleanup on failure in
 background thread function
Message-ID: <5ce38e57.1c69fb81.78e2f.6e07@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cffi-libs
Changeset: r96652:39c032729bb6
Date: 2019-05-21 08:35 +0300
http://bitbucket.org/pypy/pypy/changeset/39c032729bb6/

Log:	nicer cleanup on failure in background thread function

diff --git a/lib-python/2.7/test/test_ftplib.py b/lib-python/2.7/test/test_ftplib.py
--- a/lib-python/2.7/test/test_ftplib.py
+++ b/lib-python/2.7/test/test_ftplib.py
@@ -234,11 +234,17 @@
     def run(self):
         self.active = True
         self.__flag.set()
-        while self.active and asyncore.socket_map:
-            self.active_lock.acquire()
-            asyncore.loop(timeout=0.1, count=1)
-            self.active_lock.release()
-        asyncore.close_all(ignore_all=True)
+        try:
+            while self.active and asyncore.socket_map:
+                self.active_lock.acquire()
+                try:
+                    asyncore.loop(timeout=0.1, count=1)
+                except:
+                    self.active_lock.release()
+                    raise
+                self.active_lock.release()
+        finally:
+            asyncore.close_all(ignore_all=True)
 
     def stop(self):
         assert self.active

From pypy.commits at gmail.com  Tue May 21 02:10:32 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 20 May 2019 23:10:32 -0700 (PDT)
Subject: [pypy-commit] pypy cffi-libs: fix error type
Message-ID: <5ce39658.1c69fb81.939ee.edce@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cffi-libs
Changeset: r96653:d97ff7da2e37
Date: 2019-05-21 09:09 +0300
http://bitbucket.org/pypy/pypy/changeset/d97ff7da2e37/

Log:	fix error type

diff --git a/lib_pypy/_cffi_ssl/_stdssl/error.py b/lib_pypy/_cffi_ssl/_stdssl/error.py
--- a/lib_pypy/_cffi_ssl/_stdssl/error.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/error.py
@@ -112,7 +112,7 @@
                     # s = obj.get_socket_or_None()
                     # XXX: Windows?
                     errno = ffi.errno
-                    return OSError(errno, os.strerror(errno))
+                    return SSLError(errno, os.strerror(errno))
                 else:
                     errtype = SSLSyscallError
                     errstr = "Some I/O error occurred"

From pypy.commits at gmail.com  Tue May 21 08:47:02 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 21 May 2019 05:47:02 -0700 (PDT)
Subject: [pypy-commit] pypy cffi-libs: use cffi for _hashlib
Message-ID: <5ce3f346.1c69fb81.6792f.b84b@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cffi-libs
Changeset: r96654:de9df3b7a7ac
Date: 2019-05-21 15:38 +0300
http://bitbucket.org/pypy/pypy/changeset/de9df3b7a7ac/

Log:	use cffi for _hashlib

diff --git a/lib_pypy/_hashlib/__init__.py b/lib_pypy/_hashlib/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_hashlib/__init__.py
@@ -0,0 +1,176 @@
+import sys
+from threading import Lock
+from _pypy_openssl import ffi, lib
+from _cffi_ssl._stdssl.utility import (_str_to_ffi_buffer, _bytes_with_len,
+        _str_from_buf)
+
+try: from __pypy__ import builtinify
+except ImportError: builtinify = lambda f: f
+
+
+def new(name, string=b''):
+    h = Hash(name)
+    h.update(string)
+    return h
+
+class Hash(object):
+
+    def __init__(self, name, copy_from=None):
+        self.ctx = ffi.NULL
+        self.name = name
+        digest_type = self.digest_type_by_name()
+        self.digest_size = lib.EVP_MD_size(digest_type)
+
+        # Allocate a lock for each HASH object.
+        # An optimization would be to not release the GIL on small requests,
+        # and use a custom lock only when needed.
+        self.lock = Lock()
+
+        ctx = lib.Cryptography_EVP_MD_CTX_new()
+        if ctx == ffi.NULL:
+            raise MemoryError
+        ctx = ffi.gc(ctx, lib.Cryptography_EVP_MD_CTX_free)
+
+        try:
+            if copy_from is not None:
+                # cpython uses EVP_MD_CTX_copy(...)
+                if not lib.EVP_MD_CTX_copy_ex(ctx, copy_from):
+                    raise ValueError
+            else:
+                # cpython uses EVP_DigestInit
+                lib.EVP_DigestInit_ex(ctx, digest_type, ffi.NULL)
+            self.ctx = ctx
+        except:
+            # no need to gc ctx! 
+            raise
+
+    def digest_type_by_name(self):
+        c_name = _str_to_ffi_buffer(self.name)
+        digest_type = lib.EVP_get_digestbyname(c_name)
+        if not digest_type:
+            raise ValueError("unknown hash function")
+        # TODO
+        return digest_type
+
+    def __repr__(self):
+        return "<%s HASH object at 0x%s>" % (self.name, id(self))
+
+    def update(self, string):
+        buf = ffi.from_buffer(string)
+        with self.lock:
+            # XXX try to not release the GIL for small requests
+            lib.EVP_DigestUpdate(self.ctx, buf, len(buf))
+
+    def copy(self):
+        """Return a copy of the hash object."""
+        with self.lock:
+            return Hash(self.name, copy_from=self.ctx)
+
+    def digest(self):
+        """Return the digest value as a string of binary data."""
+        return self._digest()
+
+    def hexdigest(self):
+        """Return the digest value as a string of hexadecimal digits."""
+        digest = self._digest()
+        hexdigits = '0123456789abcdef'
+        result = []
+        for c in digest:
+            result.append(hexdigits[(c >> 4) & 0xf])
+            result.append(hexdigits[ c       & 0xf])
+        return ''.join(result)
+
+    @property
+    def block_size(self):
+        return lib.EVP_MD_CTX_block_size(self.ctx)
+
+    def _digest(self):
+        ctx = lib.Cryptography_EVP_MD_CTX_new()
+        if ctx == ffi.NULL:
+            raise MemoryError
+        try:
+            with self.lock:
+                if not lib.EVP_MD_CTX_copy_ex(ctx, self.ctx):
+                    raise ValueError
+            digest_size = self.digest_size
+            buf = ffi.new("unsigned char[]", digest_size)
+            lib.EVP_DigestFinal_ex(ctx, buf, ffi.NULL)
+            return _bytes_with_len(buf, digest_size)
+        finally:
+            lib.Cryptography_EVP_MD_CTX_free(ctx)
+
+algorithms = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512')
+
+class NameFetcher:
+    def __init__(self):
+        self.meth_names = []
+        self.error = None
+
+
+def _fetch_names():
+    name_fetcher = NameFetcher()
+    handle = ffi.new_handle(name_fetcher)
+    lib.OBJ_NAME_do_all(lib.OBJ_NAME_TYPE_MD_METH, hash_name_mapper_callback, handle)
+    if name_fetcher.error:
+        raise name_fetcher.error
+    meth_names = name_fetcher.meth_names
+    name_fetcher.meth_names = None
+    return frozenset(meth_names)
+
+ at ffi.callback("void(OBJ_NAME*, void*)")
+def hash_name_mapper_callback(obj_name, userdata):
+    if not obj_name:
+        return
+    name_fetcher = ffi.from_handle(userdata)
+    # Ignore aliased names, they pollute the list and OpenSSL appears
+    # to have a its own definition of alias as the resulting list
+    # still contains duplicate and alternate names for several
+    # algorithms.
+    if obj_name.alias != 0:
+        return
+    name = _str_from_buf(obj_name.name)
+    name_fetcher.meth_names.append(name)
+
+openssl_md_meth_names = _fetch_names()
+del _fetch_names
+
+# shortcut functions
+def make_new_hash(name, funcname):
+    @builtinify
+    def new_hash(string=b''):
+        return new(name, string)
+    new_hash.__name__ = funcname
+    return new_hash
+
+for _name in algorithms:
+    _newname = 'openssl_%s' % (_name,)
+    globals()[_newname] = make_new_hash(_name, _newname)
+
+if hasattr(lib, 'PKCS5_PBKDF2_HMAC'):
+    @builtinify
+    def pbkdf2_hmac(hash_name, password, salt, iterations, dklen=None):
+        if not isinstance(hash_name, str):
+            raise TypeError("expected 'str' for name, but got %s" % type(hash_name))
+        c_name = _str_to_ffi_buffer(hash_name)
+        digest = lib.EVP_get_digestbyname(c_name)
+        if digest == ffi.NULL:
+            raise ValueError("unsupported hash type")
+        if dklen is None:
+            dklen = lib.EVP_MD_size(digest)
+        if dklen < 1:
+            raise ValueError("key length must be greater than 0.")
+        if dklen >= sys.maxsize:
+            raise OverflowError("key length is too great.")
+        if iterations < 1:
+            raise ValueError("iteration value must be greater than 0.")
+        if iterations >= sys.maxsize:
+            raise OverflowError("iteration value is too great.")
+        buf = ffi.new("unsigned char[]", dklen)
+        c_password = ffi.from_buffer(bytes(password))
+        c_salt = ffi.from_buffer(bytes(salt))
+        r = lib.PKCS5_PBKDF2_HMAC(c_password, len(c_password),
+                ffi.cast("unsigned char*",c_salt), len(c_salt),
+                iterations, digest, dklen, buf)
+        if r == 0:
+            raise ValueError
+        return _bytes_with_len(buf, dklen)
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -32,12 +32,12 @@
 working_modules.update([
     "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd",
     "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios",
-    "zlib", "bz2", "struct", "_hashlib", "_md5", "_sha", "_minimal_curses",
+    "zlib", "bz2", "struct", "_md5", "_sha", "_minimal_curses",
     "cStringIO", "thread", "itertools", "pyexpat", "cpyext", "array",
     "binascii", "_multiprocessing", '_warnings', "_collections",
     "_multibytecodec", "micronumpy", "_continuation", "_cffi_backend",
     "_csv", "_cppyy", "_pypyjson", "_jitlog",
-    # _ssl
+    #" _ssl", "_hashlib"
 ])
 
 import rpython.rlib.rvmprof.cintf

From pypy.commits at gmail.com  Tue May 21 08:47:05 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 21 May 2019 05:47:05 -0700 (PDT)
Subject: [pypy-commit] pypy cffi-libs: adapt file copied from py3.6 for
 python2
Message-ID: <5ce3f349.1c69fb81.42ec3.02df@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cffi-libs
Changeset: r96655:ca639af9285d
Date: 2019-05-21 15:45 +0300
http://bitbucket.org/pypy/pypy/changeset/ca639af9285d/

Log:	adapt file copied from py3.6 for python2

diff --git a/lib_pypy/_hashlib/__init__.py b/lib_pypy/_hashlib/__init__.py
--- a/lib_pypy/_hashlib/__init__.py
+++ b/lib_pypy/_hashlib/__init__.py
@@ -56,7 +56,10 @@
         return "<%s HASH object at 0x%s>" % (self.name, id(self))
 
     def update(self, string):
-        buf = ffi.from_buffer(string)
+        if isinstance(string, unicode):
+            buf = ffi.from_buffer(string.encode('utf-8'))
+        else:
+            buf = ffi.from_buffer(string)
         with self.lock:
             # XXX try to not release the GIL for small requests
             lib.EVP_DigestUpdate(self.ctx, buf, len(buf))
@@ -76,8 +79,8 @@
         hexdigits = '0123456789abcdef'
         result = []
         for c in digest:
-            result.append(hexdigits[(c >> 4) & 0xf])
-            result.append(hexdigits[ c       & 0xf])
+            result.append(hexdigits[(ord(c) >> 4) & 0xf])
+            result.append(hexdigits[ ord(c)       & 0xf])
         return ''.join(result)
 
     @property

From pypy.commits at gmail.com  Wed May 22 10:29:28 2019
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 22 May 2019 07:29:28 -0700 (PDT)
Subject: [pypy-commit] pypy default: remove test that is obsolete since
 a7bddff6874e
Message-ID: <5ce55cc8.1c69fb81.36ae1.71fe@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r96657:47cd0afba559
Date: 2019-05-22 15:28 +0100
http://bitbucket.org/pypy/pypy/changeset/47cd0afba559/

Log:	remove test that is obsolete since a7bddff6874e

diff --git a/pypy/tool/pytest/test/test_appsupport.py b/pypy/tool/pytest/test/test_appsupport.py
--- a/pypy/tool/pytest/test/test_appsupport.py
+++ b/pypy/tool/pytest/test/test_appsupport.py
@@ -35,25 +35,6 @@
     ])
 
 class TestSpaceConfig:
-    def test_applevel_skipped_on_cpython_and_spaceconfig(self, testdir):
-        setpypyconftest(testdir)
-        testdir.makepyfile("""
-            class AppTestClass:
-                spaceconfig = {"objspace.usemodules._random": True}
-                def setup_class(cls):
-                    assert 0
-                def test_applevel(self):
-                    pass
-        """)
-        result = testdir.runpytest("-A")
-        assert result.ret == 0
-        if hasattr(sys, 'pypy_translation_info') and \
-           sys.pypy_translation_info.get('objspace.usemodules._random'):
-            result.stdout.fnmatch_lines(["*1 error*"])
-        else:
-            # setup_class didn't get called, otherwise it would error
-            result.stdout.fnmatch_lines(["*1 skipped*"])
-
     def test_interp_spaceconfig(self, testdir):
         setpypyconftest(testdir)
         p = testdir.makepyfile("""

From pypy.commits at gmail.com  Wed May 22 11:41:47 2019
From: pypy.commits at gmail.com (mattip)
Date: Wed, 22 May 2019 08:41:47 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: issue #3017 port bpo-29097 from cpython
 fixing datetime.fromtimestamp for win32
Message-ID: <5ce56dbb.1c69fb81.2ca83.f5d0@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r96658:194165a1cae6
Date: 2019-05-22 18:40 +0300
http://bitbucket.org/pypy/pypy/changeset/194165a1cae6/

Log:	issue #3017 port bpo-29097 from cpython fixing
	datetime.fromtimestamp for win32

diff --git a/lib-python/3/datetime.py b/lib-python/3/datetime.py
--- a/lib-python/3/datetime.py
+++ b/lib-python/3/datetime.py
@@ -1456,6 +1456,14 @@
             # 23 hours at 1969-09-30 13:00:00 in Kwajalein.
             # Let's probe 24 hours in the past to detect a transition:
             max_fold_seconds = 24 * 3600
+
+            # On Windows localtime_s throws an OSError for negative values,
+            # thus we can't perform fold detection for values of time less
+            # than the max time fold. See comments in _datetimemodule's
+            # version of this method for more details.
+            if t < max_fold_seconds and sys.platform.startswith("win"):
+                return result
+
             y, m, d, hh, mm, ss = converter(t - max_fold_seconds)[:6]
             probe1 = cls(y, m, d, hh, mm, ss, us, tz)
             trans = result - probe1 - timedelta(0, max_fold_seconds)

From pypy.commits at gmail.com  Wed May 22 12:00:14 2019
From: pypy.commits at gmail.com (mattip)
Date: Wed, 22 May 2019 09:00:14 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: fix 194165a1cae6
Message-ID: <5ce5720e.1c69fb81.bb2e1.1b68@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r96659:e9bfcbb9043c
Date: 2019-05-22 18:59 +0300
http://bitbucket.org/pypy/pypy/changeset/e9bfcbb9043c/

Log:	fix 194165a1cae6

diff --git a/lib-python/3/datetime.py b/lib-python/3/datetime.py
--- a/lib-python/3/datetime.py
+++ b/lib-python/3/datetime.py
@@ -6,6 +6,7 @@
 
 import time as _time
 import math as _math
+import sys
 
 # for cpyext, use these as base classes
 from __pypy__._pypydatetime import dateinterop, deltainterop, timeinterop
@@ -1379,7 +1380,8 @@
 
     def __new__(cls, year, month=None, day=None, hour=0, minute=0, second=0,
                 microsecond=0, tzinfo=None, *, fold=0):
-        if isinstance(year, bytes) and len(year) == 10 and 1 <= year[2]&0x7F <= 12:
+        if (isinstance(year, (bytes, str)) and len(year) == 10 and
+            1 <= ord(year[2:3])&0x7F <= 12):
             # Pickle support
             self = dateinterop.__new__(cls)
             self.__setstate(year, month)

From pypy.commits at gmail.com  Wed May 22 13:53:30 2019
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 22 May 2019 10:53:30 -0700 (PDT)
Subject: [pypy-commit] pypy optimizeopt-cleanup: kill
 BaseTest._do_optimize_loop()
Message-ID: <5ce58c9a.1c69fb81.ac855.4362@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: optimizeopt-cleanup
Changeset: r96660:56e4e21e99d2
Date: 2019-05-17 02:34 +0100
http://bitbucket.org/pypy/pypy/changeset/56e4e21e99d2/

Log:	kill BaseTest._do_optimize_loop()

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -34,8 +34,9 @@
         call_pure_results = self._convert_call_pure_results(call_pure_results)
         trace = convert_loop_to_trace(loop, self.metainterp_sd)
         compile_data = compile.SimpleCompileData(
-            trace, call_pure_results=call_pure_results)
-        info, ops = self._do_optimize_loop(compile_data)
+            trace, call_pure_results=call_pure_results,
+            enable_opts=self.enable_opts)
+        info, ops = compile_data.optimize_trace(self.metainterp_sd, None, {})
         label_op = ResOperation(rop.LABEL, info.inputargs)
         loop.inputargs = info.inputargs
         loop.operations = [label_op] + ops
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py
@@ -33,7 +33,7 @@
             self.convert_values(bridge.operations[-1].getarglist(), bridge_values),
             None, enable_opts=self.enable_opts,
             inline_short_preamble=inline_short_preamble)
-        bridge_info, ops = self._do_optimize_loop(data)
+        bridge_info, ops = data.optimize_trace(self.metainterp_sd, None, {})
         loop.check_consistency(check_descr=False)
         info.preamble.check_consistency(check_descr=False)
         bridge.operations = ([ResOperation(rop.LABEL, bridge_info.inputargs)] +
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
@@ -42,8 +42,10 @@
         inputargs = loop.inputargs
         preamble = TreeLoop('preamble')
         trace = oparser.convert_loop_to_trace(loop, self.metainterp_sd)
-        compile_data = PreambleCompileData(trace, inputargs)
-        start_state, newops = self._do_optimize_loop(compile_data)
+        compile_data = PreambleCompileData(
+            trace, inputargs, enable_opts=self.enable_opts)
+        start_state, newops = compile_data.optimize_trace(
+            self.metainterp_sd, None, {})
         preamble.operations = newops
         preamble.inputargs = start_state.renamed_inputargs
         return start_state, loop, preamble
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -544,11 +544,6 @@
         assert equaloplists(optimized.operations,
                             expected.operations, False, remap, text_right)
 
-    def _do_optimize_loop(self, compile_data):
-        compile_data.enable_opts = self.enable_opts
-        state = compile_data.optimize_trace(self.metainterp_sd, None, {})
-        return state
-
     def _convert_call_pure_results(self, d):
         if d is None:
             return
@@ -586,11 +581,13 @@
         t = convert_loop_to_trace(loop, self.metainterp_sd)
         preamble_data = compile.PreambleCompileData(
             t, runtime_boxes, call_pure_results, enable_opts=self.enable_opts)
-        start_state, preamble_ops = self._do_optimize_loop(preamble_data)
+        start_state, preamble_ops = preamble_data.optimize_trace(
+            self.metainterp_sd, None, {})
         preamble_data.forget_optimization_info()
-        loop_data = compile.UnrolledLoopData(preamble_data.trace,
-            celltoken, start_state, call_pure_results)
-        loop_info, ops = self._do_optimize_loop(loop_data)
+        loop_data = compile.UnrolledLoopData(
+            preamble_data.trace, celltoken, start_state, call_pure_results,
+            enable_opts=self.enable_opts)
+        loop_info, ops = loop_data.optimize_trace(self.metainterp_sd, None, {})
         preamble = TreeLoop('preamble')
         preamble.inputargs = start_state.renamed_inputargs
         start_label = ResOperation(rop.LABEL, start_state.renamed_inputargs)
diff --git a/rpython/jit/metainterp/test/test_resume.py b/rpython/jit/metainterp/test/test_resume.py
--- a/rpython/jit/metainterp/test/test_resume.py
+++ b/rpython/jit/metainterp/test/test_resume.py
@@ -1544,4 +1544,3 @@
     assert not modifier._invalidation_needed(10, 2)
     assert not modifier._invalidation_needed(10, 3)
     assert modifier._invalidation_needed(10, 4)
-

From pypy.commits at gmail.com  Wed May 22 13:53:32 2019
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 22 May 2019 10:53:32 -0700 (PDT)
Subject: [pypy-commit] pypy optimizeopt-cleanup: Remove dead code
Message-ID: <5ce58c9c.1c69fb81.48087.4eb8@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: optimizeopt-cleanup
Changeset: r96661:0bfaaf883296
Date: 2019-05-22 18:52 +0100
http://bitbucket.org/pypy/pypy/changeset/0bfaaf883296/

Log:	Remove dead code

diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -1,27 +1,19 @@
-import os
 from collections import OrderedDict
 
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.jit.metainterp.optimizeopt.util import args_dict
-from rpython.jit.metainterp.history import Const, ConstInt, new_ref_dict
-from rpython.jit.metainterp.jitexc import JitException
+from rpython.jit.metainterp.history import new_ref_dict
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimization, REMOVED
 from rpython.jit.metainterp.optimizeopt.util import (
     make_dispatcher_method, get_box_replacement)
 from rpython.jit.metainterp.optimizeopt.intutils import IntBound
 from rpython.jit.metainterp.optimizeopt.shortpreamble import PreambleOp
 from rpython.jit.metainterp.optimize import InvalidLoop
-from rpython.jit.metainterp.resoperation import rop, ResOperation, OpHelpers,\
-     GuardResOp
+from rpython.jit.metainterp.resoperation import rop
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.jit.metainterp.optimizeopt import info
 
 
-
-class BogusImmutableField(JitException):
-    pass
-
-
 class AbstractCachedEntry(object):
     """ abstract base class abstracting over the difference between caching
     struct fields and array items. """
@@ -133,7 +125,6 @@
         elif not can_cache:
             self.invalidate(descr)
 
-
     # abstract methods
 
     def _get_rhs_from_set_op(self, op):
@@ -557,13 +548,6 @@
 
     def optimize_SETFIELD_GC(self, op):
         self.setfield(op)
-        #opnum = OpHelpers.getfield_pure_for_descr(op.getdescr())
-        #if self.has_pure_result(opnum, [op.getarg(0)],
-        #                        op.getdescr()):
-        #    os.write(2, '[bogus _immutable_field_ declaration: %s]\n' %
-        #             (op.getdescr().repr_of_descr()))
-        #    raise BogusImmutableField
-        #
 
     def setfield(self, op):
         cf = self.field_cache(op.getdescr())
@@ -632,13 +616,6 @@
     optimize_GETARRAYITEM_GC_PURE_F = optimize_GETARRAYITEM_GC_PURE_I
 
     def optimize_SETARRAYITEM_GC(self, op):
-        #opnum = OpHelpers.getarrayitem_pure_for_descr(op.getdescr())
-        #if self.has_pure_result(opnum, [op.getarg(0), op.getarg(1)],
-        #                        op.getdescr()):
-        #    os.write(2, '[bogus immutable array declaration: %s]\n' %
-        #             (op.getdescr().repr_of_descr()))
-        #    raise BogusImmutableField
-        #
         indexb = self.getintbound(op.getarg(1))
         if indexb.is_constant():
             arrayinfo = self.ensure_ptr_info_arg0(op)
@@ -693,10 +670,10 @@
         result_getfield = []
         for descr, cf in self.cached_fields.iteritems():
             if cf._lazy_set:
-                continue # XXX safe default for now
+                continue  # XXX safe default for now
             parent_descr = descr.get_parent_descr()
             if not parent_descr.is_object():
-                continue # XXX could be extended to non-instance objects
+                continue  # XXX could be extended to non-instance objects
             for i, box1 in enumerate(cf.cached_structs):
                 if not box1.is_constant() and box1 not in available_boxes:
                     continue
@@ -713,7 +690,7 @@
         for descr, indexdict in self.cached_arrayitems.iteritems():
             for index, cf in indexdict.iteritems():
                 if cf._lazy_set:
-                    continue # XXX safe default for now
+                    continue  # XXX safe default for now
                 for i, box1 in enumerate(cf.cached_structs):
                     if not box1.is_constant() and box1 not in available_boxes:
                         continue
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -186,11 +186,6 @@
         if self.optimizer.optpure:
             self.optimizer.optpure.pure_from_args(opnum, args, op, descr)
 
-    def has_pure_result(self, opnum, args, descr):
-        if self.optimizer.optpure:
-            return self.optimizer.optpure.has_pure_result(opnum, args, descr)
-        return False
-
     def get_pure_result(self, key):
         if self.optimizer.optpure:
             return self.optimizer.optpure.get_pure_result(key)
diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -245,10 +245,6 @@
         newop.set_forwarded(op)
         self.pure(opnum, newop)
 
-    def has_pure_result(self, opnum, args, descr):
-        return False
-    # XXX
-
     def get_pure_result(self, op):
         recentops = self.getrecentops(op.getopnum())
         return recentops.lookup(self.optimizer, op)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -5015,18 +5015,6 @@
         """
         self.optimize_loop(ops, expected)
 
-    def test_complains_getfieldpure_setfield(self):
-        from rpython.jit.metainterp.optimizeopt.heap import BogusImmutableField
-        py.test.skip("disabled for now")
-        ops = """
-        [p3]
-        p1 = escape_r()
-        p2 = getfield_gc_r(p1, descr=nextdescr)
-        setfield_gc(p1, p3, descr=nextdescr)
-        jump(p3)
-        """
-        self.raises(BogusImmutableField, self.optimize_loop, ops, "crash!")
-
     def test_dont_complains_different_field(self):
         ops = """
         [p3]

From pypy.commits at gmail.com  Thu May 23 11:37:22 2019
From: pypy.commits at gmail.com (mattip)
Date: Thu, 23 May 2019 08:37:22 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: fix win32 only test, add check for bytes
Message-ID: <5ce6be32.1c69fb81.f9d35.3e85@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r96663:41fb5a04a33e
Date: 2019-05-23 08:36 -0700
http://bitbucket.org/pypy/pypy/changeset/41fb5a04a33e/

Log:	fix win32 only test, add check for bytes

diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -1462,7 +1462,7 @@
         with open(fname, "w") as f:
             f.write("this is a rename test")
         str_name = str(self.pdir) + '/test_rename.txt'
-        os.rename(self.path, str_name)
+        os.rename(fname, str_name)
         with open(str_name) as f:
             assert f.read() == 'this is a rename test'
         os.rename(str_name, fname)
@@ -1471,6 +1471,11 @@
         with open(unicode_name) as f:
             assert f.read() == 'this is a rename test'
         os.rename(unicode_name, fname)
+        
+        os.rename(bytes(fname, 'utf-8'), bytes(str_name, 'utf-8'))
+        with open(str_name) as f:
+            assert f.read() == 'this is a rename test'
+        os.rename(str_name, fname)
         with open(fname) as f:
             assert f.read() == 'this is a rename test'
         os.unlink(fname)

From pypy.commits at gmail.com  Fri May 24 01:07:34 2019
From: pypy.commits at gmail.com (mattip)
Date: Thu, 23 May 2019 22:07:34 -0700 (PDT)
Subject: [pypy-commit] pypy cffi-libs: close branch to be merged
Message-ID: <5ce77c16.1c69fb81.864fc.cd47@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cffi-libs
Changeset: r96664:64df157240e6
Date: 2019-05-24 07:41 +0300
http://bitbucket.org/pypy/pypy/changeset/64df157240e6/

Log:	close branch to be merged

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -25,3 +25,8 @@
 .. branch: shadowstack-issue2722
 
 Make the shadowstack size more dynamic
+
+.. branch: cffi-libs
+
+Move _ssl and _hashlib from rpython to a cffi-based module, like on python3.
+Reduces the number of problematic linked-in libraries (libssl, libcrypto)

From pypy.commits at gmail.com  Fri May 24 01:07:37 2019
From: pypy.commits at gmail.com (mattip)
Date: Thu, 23 May 2019 22:07:37 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge cffi-libs,
 which moves _ssl and _hashlib to cffi-based implementations
Message-ID: <5ce77c19.1c69fb81.9be70.4e9c@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r96665:50956a7107b8
Date: 2019-05-24 07:42 +0300
http://bitbucket.org/pypy/pypy/changeset/50956a7107b8/

Log:	merge cffi-libs, which moves _ssl and _hashlib to cffi-based
	implementations

diff too long, truncating to 2000 out of 11657 lines

diff --git a/lib-python/2.7/test/test_ftplib.py b/lib-python/2.7/test/test_ftplib.py
--- a/lib-python/2.7/test/test_ftplib.py
+++ b/lib-python/2.7/test/test_ftplib.py
@@ -234,11 +234,17 @@
     def run(self):
         self.active = True
         self.__flag.set()
-        while self.active and asyncore.socket_map:
-            self.active_lock.acquire()
-            asyncore.loop(timeout=0.1, count=1)
-            self.active_lock.release()
-        asyncore.close_all(ignore_all=True)
+        try:
+            while self.active and asyncore.socket_map:
+                self.active_lock.acquire()
+                try:
+                    asyncore.loop(timeout=0.1, count=1)
+                except:
+                    self.active_lock.release()
+                    raise
+                self.active_lock.release()
+        finally:
+            asyncore.close_all(ignore_all=True)
 
     def stop(self):
         assert self.active
diff --git a/lib_pypy/_cffi_ssl/LICENSE b/lib_pypy/_cffi_ssl/LICENSE
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/LICENSE
@@ -0,0 +1,26 @@
+
+Except when otherwise stated (look for LICENSE files in directories or
+information at the beginning of each file) all software and
+documentation is licensed as follows: 
+
+    The MIT License
+
+    Permission is hereby granted, free of charge, to any person 
+    obtaining a copy of this software and associated documentation 
+    files (the "Software"), to deal in the Software without 
+    restriction, including without limitation the rights to use, 
+    copy, modify, merge, publish, distribute, sublicense, and/or 
+    sell copies of the Software, and to permit persons to whom the 
+    Software is furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included 
+    in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+    OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
+    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+    DEALINGS IN THE SOFTWARE.
+
diff --git a/lib_pypy/_cffi_ssl/README.md b/lib_pypy/_cffi_ssl/README.md
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/README.md
@@ -0,0 +1,35 @@
+# PyPy's SSL module
+
+All of the CFFI code is copied from cryptography, wich patches contributed
+back to cryptography. PyPy vendors it's own copy of the cffi backend thus
+it renames the compiled shared object to _pypy_openssl.so (which means
+that cryptography can ship their own cffi backend)
+
+NOTE: currently, we have the following changes:
+
+* ``_cffi_src/openssl/callbacks.py`` to not rely on the CPython C API
+  (this change is now backported)
+
+* ``_cffi_src/utils.py`` for issue #2575 (29c9a89359e4)
+
+* ``_cffi_src/openssl/x509_vfy.py`` for issue #2605 (ca4d0c90f5a1)
+
+* ``_cffi_src/openssl/pypy_win32_extra.py`` for Win32-only functionality like ssl.enum_certificates()
+
+
+# Tests?
+
+Currently this module is tested using CPython's standard library test suite.
+
+# Install it into PyPy's source tree
+
+Copy over all the sources into the folder `lib_pypy/_cffi_ssl/*`. Updating the cffi backend can be simply done by the following command::
+
+    $ cp -r <cloned cryptography folder>/src/_cffi_src/* .
+
+NOTE: you need to keep our version of ``_cffi_src/openssl/callbacks.py``
+for now!
+
+# Crpytography version
+
+Copied over release version `1.7.2`
diff --git a/lib_pypy/_cffi_ssl/__init__.py b/lib_pypy/_cffi_ssl/__init__.py
new file mode 100644
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/__init__.py b/lib_pypy/_cffi_ssl/_cffi_src/__init__.py
new file mode 100644
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/build_commoncrypto.py b/lib_pypy/_cffi_ssl/_cffi_src/build_commoncrypto.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/build_commoncrypto.py
@@ -0,0 +1,33 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+from _cffi_src.utils import build_ffi_for_binding
+
+
+ffi = build_ffi_for_binding(
+    module_name="_commoncrypto",
+    module_prefix="_cffi_src.commoncrypto.",
+    modules=[
+        "cf",
+        "common_digest",
+        "common_hmac",
+        "common_key_derivation",
+        "common_cryptor",
+        "common_symmetric_key_wrap",
+        "seccertificate",
+        "secimport",
+        "secitem",
+        "seckey",
+        "seckeychain",
+        "secpolicy",
+        "sectransform",
+        "sectrust",
+        "secure_transport",
+    ],
+    extra_link_args=[
+        "-framework", "Security", "-framework", "CoreFoundation"
+    ],
+)
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/build_constant_time.py b/lib_pypy/_cffi_ssl/_cffi_src/build_constant_time.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/build_constant_time.py
@@ -0,0 +1,27 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+import os
+
+from _cffi_src.utils import build_ffi, compiler_type, extra_link_args
+
+
+with open(os.path.join(
+    os.path.dirname(__file__), "hazmat_src/constant_time.h"
+)) as f:
+    types = f.read()
+
+with open(os.path.join(
+    os.path.dirname(__file__), "hazmat_src/constant_time.c"
+)) as f:
+    functions = f.read()
+
+ffi = build_ffi(
+    module_name="_constant_time",
+    cdef_source=types,
+    verify_source=functions,
+    extra_link_args=extra_link_args(compiler_type()),
+)
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/build_openssl.py b/lib_pypy/_cffi_ssl/_cffi_src/build_openssl.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/build_openssl.py
@@ -0,0 +1,86 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+import os
+import sys
+
+from _cffi_src.utils import (
+    build_ffi_for_binding, compiler_type, extra_link_args
+)
+
+
+def _get_openssl_libraries(platform):
+    # OpenSSL goes by a different library name on different operating systems.
+    if platform == "darwin":
+        return _osx_libraries(
+            os.environ.get("CRYPTOGRAPHY_OSX_NO_LINK_FLAGS")
+        )
+    elif platform == "win32":
+        if compiler_type() == "msvc":
+            libs = ["libeay32", "ssleay32"]
+        else:
+            libs = ["ssl", "crypto"]
+        return libs + ["advapi32", "crypt32", "gdi32", "user32", "ws2_32"]
+    else:
+        # In some circumstances, the order in which these libs are
+        # specified on the linker command-line is significant;
+        # libssl must come before libcrypto
+        # (http://marc.info/?l=openssl-users&m=135361825921871)
+        return ["ssl", "crypto"]
+
+
+def _osx_libraries(build_static):
+    # For building statically we don't want to pass the -lssl or -lcrypto flags
+    if build_static == "1":
+        return []
+    else:
+        return ["ssl", "crypto"]
+
+
+ffi = build_ffi_for_binding(
+    module_name="_openssl",
+    module_prefix="_cffi_src.openssl.",
+    modules=[
+        # This goes first so we can define some cryptography-wide symbols.
+        "cryptography",
+
+        "aes",
+        "asn1",
+        "bignum",
+        "bio",
+        "cmac",
+        "cms",
+        "conf",
+        "crypto",
+        "dh",
+        "dsa",
+        "ec",
+        "ecdh",
+        "ecdsa",
+        "engine",
+        "err",
+        "evp",
+        "hmac",
+        "nid",
+        "objects",
+        "ocsp",
+        "opensslv",
+        "osrandom_engine",
+        "pem",
+        "pkcs12",
+        "rand",
+        "rsa",
+        "ssl",
+        "x509",
+        "x509name",
+        "x509v3",
+        "x509_vfy",
+        "pkcs7",
+        "callbacks",
+    ],
+    libraries=_get_openssl_libraries(sys.platform),
+    extra_link_args=extra_link_args(compiler_type()),
+)
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/build_padding.py b/lib_pypy/_cffi_ssl/_cffi_src/build_padding.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/build_padding.py
@@ -0,0 +1,27 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+import os
+
+from _cffi_src.utils import build_ffi, compiler_type, extra_link_args
+
+
+with open(os.path.join(
+    os.path.dirname(__file__), "hazmat_src/padding.h"
+)) as f:
+    types = f.read()
+
+with open(os.path.join(
+    os.path.dirname(__file__), "hazmat_src/padding.c"
+)) as f:
+    functions = f.read()
+
+ffi = build_ffi(
+    module_name="_padding",
+    cdef_source=types,
+    verify_source=functions,
+    extra_link_args=extra_link_args(compiler_type()),
+)
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/__init__.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/__init__.py
@@ -0,0 +1,5 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/cf.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/cf.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/cf.py
@@ -0,0 +1,113 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CoreFoundation/CoreFoundation.h>
+"""
+
+TYPES = """
+typedef bool Boolean;
+typedef signed long OSStatus;
+typedef unsigned char UInt8;
+typedef uint32_t UInt32;
+
+typedef const void * CFAllocatorRef;
+const CFAllocatorRef kCFAllocatorDefault;
+typedef ... *CFDataRef;
+typedef signed long long CFIndex;
+typedef ... *CFStringRef;
+typedef ... *CFArrayRef;
+typedef ... *CFMutableArrayRef;
+typedef ... *CFBooleanRef;
+typedef ... *CFErrorRef;
+typedef ... *CFNumberRef;
+typedef ... *CFTypeRef;
+typedef ... *CFDictionaryRef;
+typedef ... *CFMutableDictionaryRef;
+typedef struct {
+    ...;
+} CFDictionaryKeyCallBacks;
+typedef struct {
+    ...;
+} CFDictionaryValueCallBacks;
+typedef struct {
+    ...;
+} CFRange;
+typedef struct {
+    ...;
+} CFArrayCallBacks;
+
+typedef UInt32 CFStringEncoding;
+enum {
+    kCFStringEncodingASCII = 0x0600
+};
+
+enum {
+   kCFNumberSInt8Type = 1,
+   kCFNumberSInt16Type = 2,
+   kCFNumberSInt32Type = 3,
+   kCFNumberSInt64Type = 4,
+   kCFNumberFloat32Type = 5,
+   kCFNumberFloat64Type = 6,
+   kCFNumberCharType = 7,
+   kCFNumberShortType = 8,
+   kCFNumberIntType = 9,
+   kCFNumberLongType = 10,
+   kCFNumberLongLongType = 11,
+   kCFNumberFloatType = 12,
+   kCFNumberDoubleType = 13,
+   kCFNumberCFIndexType = 14,
+   kCFNumberNSIntegerType = 15,
+   kCFNumberCGFloatType = 16,
+   kCFNumberMaxType = 16
+};
+typedef int CFNumberType;
+
+const CFDictionaryKeyCallBacks kCFTypeDictionaryKeyCallBacks;
+const CFDictionaryValueCallBacks kCFTypeDictionaryValueCallBacks;
+
+const CFArrayCallBacks kCFTypeArrayCallBacks;
+
+const CFBooleanRef kCFBooleanTrue;
+const CFBooleanRef kCFBooleanFalse;
+"""
+
+FUNCTIONS = """
+CFDataRef CFDataCreate(CFAllocatorRef, const UInt8 *, CFIndex);
+CFStringRef CFStringCreateWithCString(CFAllocatorRef, const char *,
+                                      CFStringEncoding);
+CFDictionaryRef CFDictionaryCreate(CFAllocatorRef, const void **,
+                                   const void **, CFIndex,
+                                   const CFDictionaryKeyCallBacks *,
+                                   const CFDictionaryValueCallBacks *);
+CFMutableDictionaryRef CFDictionaryCreateMutable(
+    CFAllocatorRef,
+    CFIndex,
+    const CFDictionaryKeyCallBacks *,
+    const CFDictionaryValueCallBacks *
+);
+void CFDictionarySetValue(CFMutableDictionaryRef, const void *, const void *);
+CFIndex CFArrayGetCount(CFArrayRef);
+const void *CFArrayGetValueAtIndex(CFArrayRef, CFIndex);
+CFIndex CFDataGetLength(CFDataRef);
+void CFDataGetBytes(CFDataRef, CFRange, UInt8 *);
+CFRange CFRangeMake(CFIndex, CFIndex);
+void CFShow(CFTypeRef);
+Boolean CFBooleanGetValue(CFBooleanRef);
+CFNumberRef CFNumberCreate(CFAllocatorRef, CFNumberType, const void *);
+void CFRelease(CFTypeRef);
+CFTypeRef CFRetain(CFTypeRef);
+
+CFMutableArrayRef CFArrayCreateMutable(CFAllocatorRef, CFIndex,
+                                       const CFArrayCallBacks *);
+void CFArrayAppendValue(CFMutableArrayRef, const void *);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_cryptor.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_cryptor.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_cryptor.py
@@ -0,0 +1,99 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonCryptor.h>
+"""
+
+TYPES = """
+enum {
+    kCCAlgorithmAES128 = 0,
+    kCCAlgorithmDES,
+    kCCAlgorithm3DES,
+    kCCAlgorithmCAST,
+    kCCAlgorithmRC4,
+    kCCAlgorithmRC2,
+    kCCAlgorithmBlowfish
+};
+typedef uint32_t CCAlgorithm;
+enum {
+    kCCSuccess = 0,
+    kCCParamError = -4300,
+    kCCBufferTooSmall = -4301,
+    kCCMemoryFailure = -4302,
+    kCCAlignmentError = -4303,
+    kCCDecodeError = -4304,
+    kCCUnimplemented = -4305
+};
+typedef int32_t CCCryptorStatus;
+typedef uint32_t CCOptions;
+enum {
+    kCCEncrypt = 0,
+    kCCDecrypt,
+};
+typedef uint32_t CCOperation;
+typedef ... *CCCryptorRef;
+
+enum {
+    kCCModeOptionCTR_LE = 0x0001,
+    kCCModeOptionCTR_BE = 0x0002
+};
+
+typedef uint32_t CCModeOptions;
+
+enum {
+    kCCModeECB = 1,
+    kCCModeCBC = 2,
+    kCCModeCFB = 3,
+    kCCModeCTR = 4,
+    kCCModeF8 = 5,
+    kCCModeLRW = 6,
+    kCCModeOFB = 7,
+    kCCModeXTS = 8,
+    kCCModeRC4 = 9,
+    kCCModeCFB8 = 10,
+    kCCModeGCM = 11
+};
+typedef uint32_t CCMode;
+enum {
+    ccNoPadding = 0,
+    ccPKCS7Padding = 1,
+};
+typedef uint32_t CCPadding;
+"""
+
+FUNCTIONS = """
+CCCryptorStatus CCCryptorCreateWithMode(CCOperation, CCMode, CCAlgorithm,
+                                        CCPadding, const void *, const void *,
+                                        size_t, const void *, size_t, int,
+                                        CCModeOptions, CCCryptorRef *);
+CCCryptorStatus CCCryptorCreate(CCOperation, CCAlgorithm, CCOptions,
+                                const void *, size_t, const void *,
+                                CCCryptorRef *);
+CCCryptorStatus CCCryptorUpdate(CCCryptorRef, const void *, size_t, void *,
+                                size_t, size_t *);
+CCCryptorStatus CCCryptorFinal(CCCryptorRef, void *, size_t, size_t *);
+CCCryptorStatus CCCryptorRelease(CCCryptorRef);
+
+CCCryptorStatus CCCryptorGCMAddIV(CCCryptorRef, const void *, size_t);
+CCCryptorStatus CCCryptorGCMAddAAD(CCCryptorRef, const void *, size_t);
+CCCryptorStatus CCCryptorGCMEncrypt(CCCryptorRef, const void *, size_t,
+                                    void *);
+CCCryptorStatus CCCryptorGCMDecrypt(CCCryptorRef, const void *, size_t,
+                                    void *);
+CCCryptorStatus CCCryptorGCMFinal(CCCryptorRef, const void *, size_t *);
+CCCryptorStatus CCCryptorGCMReset(CCCryptorRef);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+/* Not defined in the public header */
+enum {
+    kCCModeGCM = 11
+};
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_digest.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_digest.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_digest.py
@@ -0,0 +1,58 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonDigest.h>
+"""
+
+TYPES = """
+typedef uint32_t CC_LONG;
+typedef uint64_t CC_LONG64;
+typedef struct CC_MD5state_st {
+    ...;
+} CC_MD5_CTX;
+typedef struct CC_SHA1state_st {
+    ...;
+} CC_SHA1_CTX;
+typedef struct CC_SHA256state_st {
+    ...;
+} CC_SHA256_CTX;
+typedef struct CC_SHA512state_st {
+    ...;
+} CC_SHA512_CTX;
+"""
+
+FUNCTIONS = """
+int CC_MD5_Init(CC_MD5_CTX *);
+int CC_MD5_Update(CC_MD5_CTX *, const void *, CC_LONG);
+int CC_MD5_Final(unsigned char *, CC_MD5_CTX *);
+
+int CC_SHA1_Init(CC_SHA1_CTX *);
+int CC_SHA1_Update(CC_SHA1_CTX *, const void *, CC_LONG);
+int CC_SHA1_Final(unsigned char *, CC_SHA1_CTX *);
+
+int CC_SHA224_Init(CC_SHA256_CTX *);
+int CC_SHA224_Update(CC_SHA256_CTX *, const void *, CC_LONG);
+int CC_SHA224_Final(unsigned char *, CC_SHA256_CTX *);
+
+int CC_SHA256_Init(CC_SHA256_CTX *);
+int CC_SHA256_Update(CC_SHA256_CTX *, const void *, CC_LONG);
+int CC_SHA256_Final(unsigned char *, CC_SHA256_CTX *);
+
+int CC_SHA384_Init(CC_SHA512_CTX *);
+int CC_SHA384_Update(CC_SHA512_CTX *, const void *, CC_LONG);
+int CC_SHA384_Final(unsigned char *, CC_SHA512_CTX *);
+
+int CC_SHA512_Init(CC_SHA512_CTX *);
+int CC_SHA512_Update(CC_SHA512_CTX *, const void *, CC_LONG);
+int CC_SHA512_Final(unsigned char *, CC_SHA512_CTX *);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_hmac.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_hmac.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_hmac.py
@@ -0,0 +1,37 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonHMAC.h>
+"""
+
+TYPES = """
+typedef struct {
+    ...;
+} CCHmacContext;
+enum {
+    kCCHmacAlgSHA1,
+    kCCHmacAlgMD5,
+    kCCHmacAlgSHA256,
+    kCCHmacAlgSHA384,
+    kCCHmacAlgSHA512,
+    kCCHmacAlgSHA224
+};
+typedef uint32_t CCHmacAlgorithm;
+"""
+
+FUNCTIONS = """
+void CCHmacInit(CCHmacContext *, CCHmacAlgorithm, const void *, size_t);
+void CCHmacUpdate(CCHmacContext *, const void *, size_t);
+void CCHmacFinal(CCHmacContext *, void *);
+
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_key_derivation.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_key_derivation.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_key_derivation.py
@@ -0,0 +1,39 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonKeyDerivation.h>
+"""
+
+TYPES = """
+enum {
+    kCCPBKDF2 = 2,
+};
+typedef uint32_t CCPBKDFAlgorithm;
+enum {
+    kCCPRFHmacAlgSHA1 = 1,
+    kCCPRFHmacAlgSHA224 = 2,
+    kCCPRFHmacAlgSHA256 = 3,
+    kCCPRFHmacAlgSHA384 = 4,
+    kCCPRFHmacAlgSHA512 = 5,
+};
+typedef uint32_t CCPseudoRandomAlgorithm;
+typedef unsigned int uint;
+"""
+
+FUNCTIONS = """
+int CCKeyDerivationPBKDF(CCPBKDFAlgorithm, const char *, size_t,
+                         const uint8_t *, size_t, CCPseudoRandomAlgorithm,
+                         uint, uint8_t *, size_t);
+uint CCCalibratePBKDF(CCPBKDFAlgorithm, size_t, size_t,
+                      CCPseudoRandomAlgorithm, size_t, uint32_t);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_symmetric_key_wrap.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_symmetric_key_wrap.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_symmetric_key_wrap.py
@@ -0,0 +1,35 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonSymmetricKeywrap.h>
+"""
+
+TYPES = """
+enum {
+    kCCWRAPAES = 1,
+};
+
+typedef uint32_t CCWrappingAlgorithm;
+"""
+
+FUNCTIONS = """
+int CCSymmetricKeyWrap(CCWrappingAlgorithm, const uint8_t *, const size_t,
+                        const uint8_t *, size_t, const uint8_t *, size_t,
+                        uint8_t *, size_t *);
+int CCSymmetricKeyUnwrap(CCWrappingAlgorithm algorithm, const uint8_t *,
+                         const size_t, const uint8_t *, size_t,
+                         const uint8_t *, size_t, uint8_t *, size_t *);
+size_t CCSymmetricWrappedSize(CCWrappingAlgorithm, size_t);
+size_t CCSymmetricUnwrappedSize(CCWrappingAlgorithm, size_t);
+
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seccertificate.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seccertificate.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seccertificate.py
@@ -0,0 +1,23 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecCertificate.h>
+"""
+
+TYPES = """
+typedef ... *SecCertificateRef;
+"""
+
+FUNCTIONS = """
+SecCertificateRef SecCertificateCreateWithData(CFAllocatorRef, CFDataRef);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secimport.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secimport.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secimport.py
@@ -0,0 +1,86 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecImportExport.h>
+"""
+
+TYPES = """
+typedef ... *SecAccessRef;
+
+CFStringRef kSecImportExportPassphrase;
+CFStringRef kSecImportExportKeychain;
+CFStringRef kSecImportExportAccess;
+
+typedef uint32_t SecExternalItemType;
+enum {
+    kSecItemTypeUnknown,
+    kSecItemTypePrivateKey,
+    kSecItemTypePublicKey,
+    kSecItemTypeSessionKey,
+    kSecItemTypeCertificate,
+    kSecItemTypeAggregate
+};
+
+
+typedef uint32_t SecExternalFormat;
+enum {
+    kSecFormatUnknown = 0,
+    kSecFormatOpenSSL,
+    kSecFormatSSH,
+    kSecFormatBSAFE,
+    kSecFormatRawKey,
+    kSecFormatWrappedPKCS8,
+    kSecFormatWrappedOpenSSL,
+    kSecFormatWrappedSSH,
+    kSecFormatWrappedLSH,
+    kSecFormatX509Cert,
+    kSecFormatPEMSequence,
+    kSecFormatPKCS7,
+    kSecFormatPKCS12,
+    kSecFormatNetscapeCertSequence,
+    kSecFormatSSHv2
+};
+
+typedef uint32_t SecItemImportExportFlags;
+enum {
+    kSecKeyImportOnlyOne        = 0x00000001,
+    kSecKeySecurePassphrase     = 0x00000002,
+    kSecKeyNoAccessControl      = 0x00000004
+};
+typedef uint32_t SecKeyImportExportFlags;
+
+typedef struct {
+    /* for import and export */
+    uint32_t version;
+    SecKeyImportExportFlags  flags;
+    CFTypeRef                passphrase;
+    CFStringRef              alertTitle;
+    CFStringRef              alertPrompt;
+
+    /* for import only */
+    SecAccessRef             accessRef;
+    CFArrayRef               keyUsage;
+
+    CFArrayRef               keyAttributes;
+} SecItemImportExportKeyParameters;
+"""
+
+FUNCTIONS = """
+OSStatus SecItemImport(CFDataRef, CFStringRef, SecExternalFormat *,
+                       SecExternalItemType *, SecItemImportExportFlags,
+                       const SecItemImportExportKeyParameters *,
+                       SecKeychainRef, CFArrayRef *);
+OSStatus SecPKCS12Import(CFDataRef, CFDictionaryRef, CFArrayRef *);
+OSStatus SecItemExport(CFTypeRef, SecExternalFormat, SecItemImportExportFlags,
+                       const SecItemImportExportKeyParameters *, CFDataRef *);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secitem.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secitem.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secitem.py
@@ -0,0 +1,27 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecItem.h>
+"""
+
+TYPES = """
+const CFTypeRef kSecAttrKeyType;
+const CFTypeRef kSecAttrKeySizeInBits;
+const CFTypeRef kSecAttrIsPermanent;
+const CFTypeRef kSecAttrKeyTypeRSA;
+const CFTypeRef kSecAttrKeyTypeDSA;
+const CFTypeRef kSecUseKeychain;
+"""
+
+FUNCTIONS = """
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seckey.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seckey.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seckey.py
@@ -0,0 +1,24 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecKey.h>
+"""
+
+TYPES = """
+typedef ... *SecKeyRef;
+"""
+
+FUNCTIONS = """
+OSStatus SecKeyGeneratePair(CFDictionaryRef, SecKeyRef *, SecKeyRef *);
+size_t SecKeyGetBlockSize(SecKeyRef);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seckeychain.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seckeychain.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seckeychain.py
@@ -0,0 +1,25 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecKeychain.h>
+"""
+
+TYPES = """
+typedef ... *SecKeychainRef;
+"""
+
+FUNCTIONS = """
+OSStatus SecKeychainCreate(const char *, UInt32, const void *, Boolean,
+                           SecAccessRef, SecKeychainRef *);
+OSStatus SecKeychainDelete(SecKeychainRef);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secpolicy.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secpolicy.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secpolicy.py
@@ -0,0 +1,23 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecPolicy.h>
+"""
+
+TYPES = """
+typedef ... *SecPolicyRef;
+"""
+
+FUNCTIONS = """
+SecPolicyRef SecPolicyCreateSSL(Boolean, CFStringRef);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/sectransform.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/sectransform.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/sectransform.py
@@ -0,0 +1,68 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecDigestTransform.h>
+#include <Security/SecSignVerifyTransform.h>
+#include <Security/SecEncryptTransform.h>
+"""
+
+TYPES = """
+typedef ... *SecTransformRef;
+
+CFStringRef kSecImportExportPassphrase;
+CFStringRef kSecImportExportKeychain;
+CFStringRef kSecImportExportAccess;
+
+CFStringRef kSecEncryptionMode;
+CFStringRef kSecEncryptKey;
+CFStringRef kSecIVKey;
+CFStringRef kSecModeCBCKey;
+CFStringRef kSecModeCFBKey;
+CFStringRef kSecModeECBKey;
+CFStringRef kSecModeNoneKey;
+CFStringRef kSecModeOFBKey;
+CFStringRef kSecOAEPEncodingParametersAttributeName;
+CFStringRef kSecPaddingKey;
+CFStringRef kSecPaddingNoneKey;
+CFStringRef kSecPaddingOAEPKey;
+CFStringRef kSecPaddingPKCS1Key;
+CFStringRef kSecPaddingPKCS5Key;
+CFStringRef kSecPaddingPKCS7Key;
+
+const CFStringRef kSecTransformInputAttributeName;
+const CFStringRef kSecTransformOutputAttributeName;
+const CFStringRef kSecTransformDebugAttributeName;
+const CFStringRef kSecTransformTransformName;
+const CFStringRef kSecTransformAbortAttributeName;
+
+CFStringRef kSecInputIsAttributeName;
+CFStringRef kSecInputIsPlainText;
+CFStringRef kSecInputIsDigest;
+CFStringRef kSecInputIsRaw;
+
+const CFStringRef kSecDigestTypeAttribute;
+const CFStringRef kSecDigestLengthAttribute;
+const CFStringRef kSecDigestMD5;
+const CFStringRef kSecDigestSHA1;
+const CFStringRef kSecDigestSHA2;
+"""
+
+FUNCTIONS = """
+Boolean SecTransformSetAttribute(SecTransformRef, CFStringRef, CFTypeRef,
+                                 CFErrorRef *);
+SecTransformRef SecDecryptTransformCreate(SecKeyRef, CFErrorRef *);
+SecTransformRef SecEncryptTransformCreate(SecKeyRef, CFErrorRef *);
+SecTransformRef SecVerifyTransformCreate(SecKeyRef, CFDataRef, CFErrorRef *);
+SecTransformRef SecSignTransformCreate(SecKeyRef, CFErrorRef *) ;
+CFTypeRef SecTransformExecute(SecTransformRef, CFErrorRef *);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/sectrust.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/sectrust.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/sectrust.py
@@ -0,0 +1,39 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecTrust.h>
+"""
+
+TYPES = """
+typedef ... *SecTrustRef;
+typedef uint32_t SecTrustResultType;
+
+enum {
+    kSecTrustResultInvalid,
+    kSecTrustResultProceed,
+    kSecTrustResultDeny,
+    kSecTrustResultUnspecified,
+    kSecTrustResultRecoverableTrustFailure,
+    kSecTrustResultFatalTrustFailure,
+    kSecTrustResultOtherError
+};
+"""
+
+FUNCTIONS = """
+OSStatus SecTrustEvaluate(SecTrustRef, SecTrustResultType *);
+OSStatus SecTrustCopyAnchorCertificates(CFArrayRef *);
+"""
+
+MACROS = """
+/* The first argument changed from CFArrayRef to CFTypeRef in 10.8, so this
+ * has to go here for compatibility.
+ */
+OSStatus SecTrustCreateWithCertificates(CFTypeRef, CFTypeRef, SecTrustRef *);
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secure_transport.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secure_transport.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secure_transport.py
@@ -0,0 +1,308 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecureTransport.h>
+"""
+
+TYPES = """
+typedef ... *SSLContextRef;
+typedef const void *SSLConnectionRef;
+
+typedef enum {
+    kSSLSessionOptionBreakOnServerAuth,
+    kSSLSessionOptionBreakOnCertRequested,
+} SSLSessionOption;
+
+typedef enum {
+    kNeverAuthenticate,
+    kAlwaysAuthenticate,
+    kTryAuthenticate
+} SSLAuthenticate;
+
+typedef enum {
+    kSSLIdle,
+    kSSLHandshake,
+    kSSLConnected,
+    kSSLClosed,
+    kSSLAborted
+} SSLSessionState;
+
+typedef enum {
+    kSSLProtocolUnknown = 0,
+    kSSLProtocol3       = 2,
+    kTLSProtocol1       = 4,
+    /* DEPRECATED on iOS */
+    kSSLProtocol2       = 1,
+    kSSLProtocol3Only   = 3,
+    kTLSProtocol1Only   = 5,
+    kSSLProtocolAll     = 6,
+} SSLProtocol;
+
+typedef UInt32 SSLCipherSuite;
+enum {
+   SSL_NULL_WITH_NULL_NULL =               0x0000,
+   SSL_RSA_WITH_NULL_MD5 =                 0x0001,
+   SSL_RSA_WITH_NULL_SHA =                 0x0002,
+   SSL_RSA_EXPORT_WITH_RC4_40_MD5 =        0x0003,
+   SSL_RSA_WITH_RC4_128_MD5 =              0x0004,
+   SSL_RSA_WITH_RC4_128_SHA =              0x0005,
+   SSL_RSA_EXPORT_WITH_RC2_CBC_40_MD5 =    0x0006,
+   SSL_RSA_WITH_IDEA_CBC_SHA =             0x0007,
+   SSL_RSA_EXPORT_WITH_DES40_CBC_SHA =     0x0008,
+   SSL_RSA_WITH_DES_CBC_SHA =              0x0009,
+   SSL_RSA_WITH_3DES_EDE_CBC_SHA =         0x000A,
+   SSL_DH_DSS_EXPORT_WITH_DES40_CBC_SHA =  0x000B,
+   SSL_DH_DSS_WITH_DES_CBC_SHA =           0x000C,
+   SSL_DH_DSS_WITH_3DES_EDE_CBC_SHA =      0x000D,
+   SSL_DH_RSA_EXPORT_WITH_DES40_CBC_SHA =  0x000E,
+   SSL_DH_RSA_WITH_DES_CBC_SHA =           0x000F,
+   SSL_DH_RSA_WITH_3DES_EDE_CBC_SHA =      0x0010,
+   SSL_DHE_DSS_EXPORT_WITH_DES40_CBC_SHA = 0x0011,
+   SSL_DHE_DSS_WITH_DES_CBC_SHA =          0x0012,
+   SSL_DHE_DSS_WITH_3DES_EDE_CBC_SHA =     0x0013,
+   SSL_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA = 0x0014,
+   SSL_DHE_RSA_WITH_DES_CBC_SHA =          0x0015,
+   SSL_DHE_RSA_WITH_3DES_EDE_CBC_SHA =     0x0016,
+   SSL_DH_anon_EXPORT_WITH_RC4_40_MD5 =    0x0017,
+   SSL_DH_anon_WITH_RC4_128_MD5 =          0x0018,
+   SSL_DH_anon_EXPORT_WITH_DES40_CBC_SHA = 0x0019,
+   SSL_DH_anon_WITH_DES_CBC_SHA =          0x001A,
+   SSL_DH_anon_WITH_3DES_EDE_CBC_SHA =     0x001B,
+   SSL_FORTEZZA_DMS_WITH_NULL_SHA =        0x001C,
+   SSL_FORTEZZA_DMS_WITH_FORTEZZA_CBC_SHA =0x001D,
+
+   /* TLS addenda using AES, per RFC 3268 */
+   TLS_RSA_WITH_AES_128_CBC_SHA      =     0x002F,
+   TLS_DH_DSS_WITH_AES_128_CBC_SHA   =     0x0030,
+   TLS_DH_RSA_WITH_AES_128_CBC_SHA   =     0x0031,
+   TLS_DHE_DSS_WITH_AES_128_CBC_SHA  =     0x0032,
+   TLS_DHE_RSA_WITH_AES_128_CBC_SHA  =     0x0033,
+   TLS_DH_anon_WITH_AES_128_CBC_SHA  =     0x0034,
+   TLS_RSA_WITH_AES_256_CBC_SHA      =     0x0035,
+   TLS_DH_DSS_WITH_AES_256_CBC_SHA   =     0x0036,
+   TLS_DH_RSA_WITH_AES_256_CBC_SHA   =     0x0037,
+   TLS_DHE_DSS_WITH_AES_256_CBC_SHA  =     0x0038,
+   TLS_DHE_RSA_WITH_AES_256_CBC_SHA  =     0x0039,
+   TLS_DH_anon_WITH_AES_256_CBC_SHA  =     0x003A,
+
+   /* ECDSA addenda, RFC 4492 */
+   TLS_ECDH_ECDSA_WITH_NULL_SHA           =    0xC001,
+   TLS_ECDH_ECDSA_WITH_RC4_128_SHA        =    0xC002,
+   TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA   =    0xC003,
+   TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA    =    0xC004,
+   TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA    =    0xC005,
+   TLS_ECDHE_ECDSA_WITH_NULL_SHA          =    0xC006,
+   TLS_ECDHE_ECDSA_WITH_RC4_128_SHA       =    0xC007,
+   TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA  =    0xC008,
+   TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA   =    0xC009,
+   TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA   =    0xC00A,
+   TLS_ECDH_RSA_WITH_NULL_SHA             =    0xC00B,
+   TLS_ECDH_RSA_WITH_RC4_128_SHA          =    0xC00C,
+   TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA     =    0xC00D,
+   TLS_ECDH_RSA_WITH_AES_128_CBC_SHA      =    0xC00E,
+   TLS_ECDH_RSA_WITH_AES_256_CBC_SHA      =    0xC00F,
+   TLS_ECDHE_RSA_WITH_NULL_SHA            =    0xC010,
+   TLS_ECDHE_RSA_WITH_RC4_128_SHA         =    0xC011,
+   TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA    =    0xC012,
+   TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA     =    0xC013,
+   TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA     =    0xC014,
+   TLS_ECDH_anon_WITH_NULL_SHA            =    0xC015,
+   TLS_ECDH_anon_WITH_RC4_128_SHA         =    0xC016,
+   TLS_ECDH_anon_WITH_3DES_EDE_CBC_SHA    =    0xC017,
+   TLS_ECDH_anon_WITH_AES_128_CBC_SHA     =    0xC018,
+   TLS_ECDH_anon_WITH_AES_256_CBC_SHA     =    0xC019,
+
+   /* TLS 1.2 addenda, RFC 5246 */
+   /* Initial state. */
+   TLS_NULL_WITH_NULL_NULL                   = 0x0000,
+
+   /* Server provided RSA certificate for key exchange. */
+   TLS_RSA_WITH_NULL_MD5                     = 0x0001,
+   TLS_RSA_WITH_NULL_SHA                     = 0x0002,
+   TLS_RSA_WITH_RC4_128_MD5                  = 0x0004,
+   TLS_RSA_WITH_RC4_128_SHA                  = 0x0005,
+   TLS_RSA_WITH_3DES_EDE_CBC_SHA             = 0x000A,
+   TLS_RSA_WITH_NULL_SHA256                  = 0x003B,
+   TLS_RSA_WITH_AES_128_CBC_SHA256           = 0x003C,
+   TLS_RSA_WITH_AES_256_CBC_SHA256           = 0x003D,
+
+   /* Server-authenticated (and optionally client-authenticated)
+      Diffie-Hellman. */
+   TLS_DH_DSS_WITH_3DES_EDE_CBC_SHA          = 0x000D,
+   TLS_DH_RSA_WITH_3DES_EDE_CBC_SHA          = 0x0010,
+   TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA         = 0x0013,
+   TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA         = 0x0016,
+   TLS_DH_DSS_WITH_AES_128_CBC_SHA256        = 0x003E,
+   TLS_DH_RSA_WITH_AES_128_CBC_SHA256        = 0x003F,
+   TLS_DHE_DSS_WITH_AES_128_CBC_SHA256       = 0x0040,
+   TLS_DHE_RSA_WITH_AES_128_CBC_SHA256       = 0x0067,
+   TLS_DH_DSS_WITH_AES_256_CBC_SHA256        = 0x0068,
+   TLS_DH_RSA_WITH_AES_256_CBC_SHA256        = 0x0069,
+   TLS_DHE_DSS_WITH_AES_256_CBC_SHA256       = 0x006A,
+   TLS_DHE_RSA_WITH_AES_256_CBC_SHA256       = 0x006B,
+
+   /* Completely anonymous Diffie-Hellman */
+   TLS_DH_anon_WITH_RC4_128_MD5              = 0x0018,
+   TLS_DH_anon_WITH_3DES_EDE_CBC_SHA         = 0x001B,
+   TLS_DH_anon_WITH_AES_128_CBC_SHA256       = 0x006C,
+   TLS_DH_anon_WITH_AES_256_CBC_SHA256       = 0x006D,
+
+   /* Addenda from rfc 5288 AES Galois Counter Mode (GCM) Cipher Suites
+      for TLS. */
+   TLS_RSA_WITH_AES_128_GCM_SHA256           = 0x009C,
+   TLS_RSA_WITH_AES_256_GCM_SHA384           = 0x009D,
+   TLS_DHE_RSA_WITH_AES_128_GCM_SHA256       = 0x009E,
+   TLS_DHE_RSA_WITH_AES_256_GCM_SHA384       = 0x009F,
+   TLS_DH_RSA_WITH_AES_128_GCM_SHA256        = 0x00A0,
+   TLS_DH_RSA_WITH_AES_256_GCM_SHA384        = 0x00A1,
+   TLS_DHE_DSS_WITH_AES_128_GCM_SHA256       = 0x00A2,
+   TLS_DHE_DSS_WITH_AES_256_GCM_SHA384       = 0x00A3,
+   TLS_DH_DSS_WITH_AES_128_GCM_SHA256        = 0x00A4,
+   TLS_DH_DSS_WITH_AES_256_GCM_SHA384        = 0x00A5,
+   TLS_DH_anon_WITH_AES_128_GCM_SHA256       = 0x00A6,
+   TLS_DH_anon_WITH_AES_256_GCM_SHA384       = 0x00A7,
+
+   /* Addenda from rfc 5289  Elliptic Curve Cipher Suites with
+      HMAC SHA-256/384. */
+   TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256   = 0xC023,
+   TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384   = 0xC024,
+   TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256    = 0xC025,
+   TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384    = 0xC026,
+   TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256     = 0xC027,
+   TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384     = 0xC028,
+   TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256      = 0xC029,
+   TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384      = 0xC02A,
+
+   /* Addenda from rfc 5289  Elliptic Curve Cipher Suites with
+      SHA-256/384 and AES Galois Counter Mode (GCM) */
+   TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256   = 0xC02B,
+   TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384   = 0xC02C,
+   TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256    = 0xC02D,
+   TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384    = 0xC02E,
+   TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256     = 0xC02F,
+   TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384     = 0xC030,
+   TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256      = 0xC031,
+   TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384      = 0xC032,
+
+   /* RFC 5746 - Secure Renegotiation */
+   TLS_EMPTY_RENEGOTIATION_INFO_SCSV         = 0x00FF,
+
+   /*
+   * Tags for SSL 2 cipher kinds that are not specified
+   * for SSL 3.
+   */
+   SSL_RSA_WITH_RC2_CBC_MD5 =              0xFF80,
+   SSL_RSA_WITH_IDEA_CBC_MD5 =             0xFF81,
+   SSL_RSA_WITH_DES_CBC_MD5 =              0xFF82,
+   SSL_RSA_WITH_3DES_EDE_CBC_MD5 =         0xFF83,
+   SSL_NO_SUCH_CIPHERSUITE =               0xFFFF
+};
+
+typedef enum {
+    kSSLClientCertNone,
+    kSSLClientCertRequested,
+    kSSLClientCertSent,
+    kSSLClientCertRejected
+} SSLClientCertificateState;
+
+enum {
+    errSSLProtocol              = -9800,
+    errSSLNegotiation           = -9801,
+    errSSLFatalAlert            = -9802,
+    errSSLWouldBlock            = -9803,
+    errSSLSessionNotFound       = -9804,
+    errSSLClosedGraceful        = -9805,
+    errSSLClosedAbort           = -9806,
+    errSSLXCertChainInvalid     = -9807,
+    errSSLBadCert               = -9808,
+    errSSLCrypto                = -9809,
+    errSSLInternal              = -9810,
+    errSSLModuleAttach          = -9811,
+    errSSLUnknownRootCert       = -9812,
+    errSSLNoRootCert            = -9813,
+    errSSLCertExpired           = -9814,
+    errSSLCertNotYetValid       = -9815,
+    errSSLClosedNoNotify        = -9816,
+    errSSLBufferOverflow        = -9817,
+    errSSLBadCipherSuite        = -9818,
+    errSSLPeerUnexpectedMsg     = -9819,
+    errSSLPeerBadRecordMac      = -9820,
+    errSSLPeerDecryptionFail    = -9821,
+    errSSLPeerRecordOverflow    = -9822,
+    errSSLPeerDecompressFail    = -9823,
+    errSSLPeerHandshakeFail     = -9824,
+    errSSLPeerBadCert           = -9825,
+    errSSLPeerUnsupportedCert   = -9826,
+    errSSLPeerCertRevoked       = -9827,
+    errSSLPeerCertExpired       = -9828,
+    errSSLPeerCertUnknown       = -9829,
+    errSSLIllegalParam          = -9830,
+    errSSLPeerUnknownCA         = -9831,
+    errSSLPeerAccessDenied      = -9832,
+    errSSLPeerDecodeError       = -9833,
+    errSSLPeerDecryptError      = -9834,
+    errSSLPeerExportRestriction = -9835,
+    errSSLPeerProtocolVersion   = -9836,
+    errSSLPeerInsufficientSecurity = -9837,
+    errSSLPeerInternalError     = -9838,
+    errSSLPeerUserCancelled     = -9839,
+    errSSLPeerNoRenegotiation   = -9840,
+    errSSLServerAuthCompleted   = -9841,
+    errSSLClientCertRequested   = -9842,
+    errSSLHostNameMismatch      = -9843,
+    errSSLConnectionRefused     = -9844,
+    errSSLDecryptionFail        = -9845,
+    errSSLBadRecordMac          = -9846,
+    errSSLRecordOverflow        = -9847,
+    errSSLBadConfiguration      = -9848,
+    errSSLLast                  = -9849     /* end of range, to be deleted */
+};
+"""
+
+FUNCTIONS = """
+OSStatus SSLSetConnection(SSLContextRef, SSLConnectionRef);
+OSStatus SSLGetConnection(SSLContextRef, SSLConnectionRef *);
+OSStatus SSLSetSessionOption(SSLContextRef, SSLSessionOption, Boolean);
+OSStatus SSLSetClientSideAuthenticate(SSLContextRef, SSLAuthenticate);
+
+OSStatus SSLHandshake(SSLContextRef);
+OSStatus SSLGetSessionState(SSLContextRef, SSLSessionState *);
+OSStatus SSLGetNegotiatedProtocolVersion(SSLContextRef, SSLProtocol *);
+OSStatus SSLSetPeerID(SSLContextRef, const void *, size_t);
+OSStatus SSLGetPeerID(SSLContextRef, const void **, size_t *);
+OSStatus SSLGetBufferedReadSize(SSLContextRef, size_t *);
+OSStatus SSLRead(SSLContextRef, void *, size_t, size_t *);
+OSStatus SSLWrite(SSLContextRef, const void *, size_t, size_t *);
+OSStatus SSLClose(SSLContextRef);
+
+OSStatus SSLGetNumberSupportedCiphers(SSLContextRef, size_t *);
+OSStatus SSLGetSupportedCiphers(SSLContextRef, SSLCipherSuite *, size_t *);
+OSStatus SSLSetEnabledCiphers(SSLContextRef, const SSLCipherSuite *, size_t);
+OSStatus SSLGetNumberEnabledCiphers(SSLContextRef, size_t *);
+OSStatus SSLGetEnabledCiphers(SSLContextRef, SSLCipherSuite *, size_t *);
+OSStatus SSLGetNegotiatedCipher(SSLContextRef, SSLCipherSuite *);
+OSStatus SSLSetDiffieHellmanParams(SSLContextRef, const void *, size_t);
+OSStatus SSLGetDiffieHellmanParams(SSLContextRef, const void **, size_t *);
+
+OSStatus SSLSetCertificateAuthorities(SSLContextRef, CFTypeRef, Boolean);
+OSStatus SSLCopyCertificateAuthorities(SSLContextRef, CFArrayRef *);
+OSStatus SSLCopyDistinguishedNames(SSLContextRef, CFArrayRef *);
+OSStatus SSLSetCertificate(SSLContextRef, CFArrayRef);
+OSStatus SSLGetClientCertificateState(SSLContextRef,
+                                      SSLClientCertificateState *);
+OSStatus SSLCopyPeerTrust(SSLContextRef, SecTrustRef *trust);
+
+OSStatus SSLSetPeerDomainName(SSLContextRef, const char *, size_t);
+OSStatus SSLGetPeerDomainNameLength(SSLContextRef, size_t *);
+OSStatus SSLGetPeerDomainName(SSLContextRef, char *, size_t *);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/constant_time.c b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/constant_time.c
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/constant_time.c
@@ -0,0 +1,22 @@
+// This file is dual licensed under the terms of the Apache License, Version
+// 2.0, and the BSD License. See the LICENSE file in the root of this
+// repository for complete details.
+
+uint8_t Cryptography_constant_time_bytes_eq(uint8_t *a, size_t len_a,
+                                            uint8_t *b, size_t len_b) {
+    size_t i = 0;
+    uint8_t mismatch = 0;
+    if (len_a != len_b) {
+        return 0;
+    }
+    for (i = 0; i < len_a; i++) {
+        mismatch |= a[i] ^ b[i];
+    }
+
+    /* Make sure any bits set are copied to the lowest bit */
+    mismatch |= mismatch >> 4;
+    mismatch |= mismatch >> 2;
+    mismatch |= mismatch >> 1;
+    /* Now check the low bit to see if it's set */
+    return (mismatch & 1) == 0;
+}
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/constant_time.h b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/constant_time.h
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/constant_time.h
@@ -0,0 +1,6 @@
+// This file is dual licensed under the terms of the Apache License, Version
+// 2.0, and the BSD License. See the LICENSE file in the root of this
+// repository for complete details.
+
+uint8_t Cryptography_constant_time_bytes_eq(uint8_t *, size_t, uint8_t *,
+                                            size_t);
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/padding.c b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/padding.c
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/padding.c
@@ -0,0 +1,65 @@
+// This file is dual licensed under the terms of the Apache License, Version
+// 2.0, and the BSD License. See the LICENSE file in the root of this
+// repository for complete details.
+
+/* Returns the value of the input with the most-significant-bit copied to all
+   of the bits. */
+static uint16_t Cryptography_DUPLICATE_MSB_TO_ALL(uint16_t a) {
+    return (1 - (a >> (sizeof(uint16_t) * 8 - 1))) - 1;
+}
+
+/* This returns 0xFFFF if a < b else 0x0000, but does so in a constant time
+   fashion */
+static uint16_t Cryptography_constant_time_lt(uint16_t a, uint16_t b) {
+    a -= b;
+    return Cryptography_DUPLICATE_MSB_TO_ALL(a);
+}
+
+uint8_t Cryptography_check_pkcs7_padding(const uint8_t *data,
+                                         uint16_t block_len) {
+    uint16_t i;
+    uint16_t pad_size = data[block_len - 1];
+    uint16_t mismatch = 0;
+    for (i = 0; i < block_len; i++) {
+        unsigned int mask = Cryptography_constant_time_lt(i, pad_size);
+        uint16_t b = data[block_len - 1 - i];
+        mismatch |= (mask & (pad_size ^ b));
+    }
+
+    /* Check to make sure the pad_size was within the valid range. */
+    mismatch |= ~Cryptography_constant_time_lt(0, pad_size);
+    mismatch |= Cryptography_constant_time_lt(block_len, pad_size);
+
+    /* Make sure any bits set are copied to the lowest bit */
+    mismatch |= mismatch >> 8;
+    mismatch |= mismatch >> 4;
+    mismatch |= mismatch >> 2;
+    mismatch |= mismatch >> 1;
+    /* Now check the low bit to see if it's set */
+    return (mismatch & 1) == 0;
+}
+
+uint8_t Cryptography_check_ansix923_padding(const uint8_t *data,
+                                            uint16_t block_len) {
+    uint16_t i;
+    uint16_t pad_size = data[block_len - 1];
+    uint16_t mismatch = 0;
+    /* Skip the first one with the pad size */
+    for (i = 1; i < block_len; i++) {
+        unsigned int mask = Cryptography_constant_time_lt(i, pad_size);
+        uint16_t b = data[block_len - 1 - i];
+        mismatch |= (mask & b);
+    }
+
+    /* Check to make sure the pad_size was within the valid range. */
+    mismatch |= ~Cryptography_constant_time_lt(0, pad_size);
+    mismatch |= Cryptography_constant_time_lt(block_len, pad_size);
+
+    /* Make sure any bits set are copied to the lowest bit */
+    mismatch |= mismatch >> 8;
+    mismatch |= mismatch >> 4;
+    mismatch |= mismatch >> 2;
+    mismatch |= mismatch >> 1;
+    /* Now check the low bit to see if it's set */
+    return (mismatch & 1) == 0;
+}
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/padding.h b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/padding.h
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/padding.h
@@ -0,0 +1,6 @@
+// This file is dual licensed under the terms of the Apache License, Version
+// 2.0, and the BSD License. See the LICENSE file in the root of this
+// repository for complete details.
+
+uint8_t Cryptography_check_pkcs7_padding(const uint8_t *, uint8_t);
+uint8_t Cryptography_check_ansix923_padding(const uint8_t *, uint8_t);
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/__init__.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/__init__.py
@@ -0,0 +1,5 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/aes.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/aes.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/aes.py
@@ -0,0 +1,50 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <openssl/aes.h>
+"""
+
+TYPES = """
+static const int Cryptography_HAS_AES_WRAP;
+static const int Cryptography_HAS_AES_CTR128_ENCRYPT;
+
+struct aes_key_st {
+    ...;
+};
+typedef struct aes_key_st AES_KEY;
+"""
+
+FUNCTIONS = """
+int AES_set_encrypt_key(const unsigned char *, const int, AES_KEY *);
+int AES_set_decrypt_key(const unsigned char *, const int, AES_KEY *);
+
+int AES_wrap_key(AES_KEY *, const unsigned char *, unsigned char *,
+                 const unsigned char *, unsigned int);
+int AES_unwrap_key(AES_KEY *, const unsigned char *, unsigned char *,
+                   const unsigned char *, unsigned int);
+"""
+
+MACROS = """
+/* The ctr128_encrypt function is only useful in 1.0.0. We can use EVP for
+   this in 1.0.1+. */
+void AES_ctr128_encrypt(const unsigned char *, unsigned char *,
+                        size_t, const AES_KEY *, unsigned char[],
+                        unsigned char[], unsigned int *);
+"""
+
+CUSTOMIZATIONS = """
+static const long Cryptography_HAS_AES_WRAP = 1;
+#if CRYPTOGRAPHY_OPENSSL_110_OR_GREATER && !defined(LIBRESSL_VERSION_NUMBER)
+static const int Cryptography_HAS_AES_CTR128_ENCRYPT = 0;
+void (*AES_ctr128_encrypt)(const unsigned char *, unsigned char *,
+                           size_t, const AES_KEY *,
+                           unsigned char[], unsigned char[],
+                           unsigned int *) = NULL;
+#else
+static const int Cryptography_HAS_AES_CTR128_ENCRYPT = 1;
+#endif
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/asn1.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/asn1.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/asn1.py
@@ -0,0 +1,164 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <openssl/asn1.h>
+"""
+
+TYPES = """
+typedef int... time_t;
+
+typedef int ASN1_BOOLEAN;
+typedef ... ASN1_INTEGER;
+
+struct asn1_string_st {
+    int length;
+    int type;
+    unsigned char *data;
+    long flags;
+};
+
+typedef struct asn1_string_st ASN1_OCTET_STRING;
+typedef struct asn1_string_st ASN1_IA5STRING;
+typedef struct asn1_string_st ASN1_BIT_STRING;
+typedef struct asn1_string_st ASN1_TIME;
+typedef ... ASN1_OBJECT;
+typedef struct asn1_string_st ASN1_STRING;
+typedef struct asn1_string_st ASN1_UTF8STRING;
+typedef ... ASN1_TYPE;
+typedef ... ASN1_GENERALIZEDTIME;
+typedef ... ASN1_ENUMERATED;
+typedef ... ASN1_ITEM;
+typedef ... ASN1_VALUE;
+
+typedef ... ASN1_ITEM_EXP;
+
+typedef ... ASN1_UTCTIME;
+
+static const int V_ASN1_GENERALIZEDTIME;
+
+static const int MBSTRING_FLAG;
+static const int MBSTRING_ASC;
+static const int MBSTRING_BMP;
+static const int MBSTRING_UTF8;
+static const int MBSTRING_UNIV;
+"""
+
+FUNCTIONS = """
+ASN1_OBJECT *ASN1_OBJECT_new(void);
+void ASN1_OBJECT_free(ASN1_OBJECT *);
+
+/*  ASN1 OBJECT IDENTIFIER */
+ASN1_OBJECT *d2i_ASN1_OBJECT(ASN1_OBJECT **, const unsigned char **, long);
+
+/*  ASN1 STRING */
+ASN1_STRING *ASN1_STRING_new(void);
+ASN1_STRING *ASN1_STRING_type_new(int);
+void ASN1_STRING_free(ASN1_STRING *);
+unsigned char *ASN1_STRING_data(ASN1_STRING *);
+int ASN1_STRING_set(ASN1_STRING *, const void *, int);
+
+/*  ASN1 OCTET STRING */
+ASN1_OCTET_STRING *ASN1_OCTET_STRING_new(void);
+void ASN1_OCTET_STRING_free(ASN1_OCTET_STRING *);
+int ASN1_OCTET_STRING_set(ASN1_OCTET_STRING *, const unsigned char *, int);
+
+/* ASN1 IA5STRING */
+ASN1_IA5STRING *ASN1_IA5STRING_new(void);
+
+/*  ASN1 INTEGER */
+ASN1_INTEGER *ASN1_INTEGER_new(void);
+void ASN1_INTEGER_free(ASN1_INTEGER *);
+int ASN1_INTEGER_set(ASN1_INTEGER *, long);
+
+/*  ASN1 TIME */
+ASN1_TIME *ASN1_TIME_new(void);
+void ASN1_TIME_free(ASN1_TIME *);
+ASN1_GENERALIZEDTIME *ASN1_TIME_to_generalizedtime(ASN1_TIME *,
+                                                   ASN1_GENERALIZEDTIME **);
+ASN1_TIME *ASN1_TIME_set(ASN1_TIME *, time_t);
+
+/*  ASN1 UTCTIME */
+ASN1_UTCTIME *ASN1_UTCTIME_new(void);
+void ASN1_UTCTIME_free(ASN1_UTCTIME *);
+int ASN1_UTCTIME_cmp_time_t(const ASN1_UTCTIME *, time_t);
+ASN1_UTCTIME *ASN1_UTCTIME_set(ASN1_UTCTIME *, time_t);
+
+/*  ASN1 GENERALIZEDTIME */
+int ASN1_GENERALIZEDTIME_set_string(ASN1_GENERALIZEDTIME *, const char *);
+ASN1_GENERALIZEDTIME *ASN1_GENERALIZEDTIME_set(ASN1_GENERALIZEDTIME *, time_t);
+void ASN1_GENERALIZEDTIME_free(ASN1_GENERALIZEDTIME *);
+int i2d_ASN1_GENERALIZEDTIME(ASN1_GENERALIZEDTIME *, unsigned char **);
+
+/*  ASN1 ENUMERATED */
+ASN1_ENUMERATED *ASN1_ENUMERATED_new(void);
+void ASN1_ENUMERATED_free(ASN1_ENUMERATED *);
+int ASN1_ENUMERATED_set(ASN1_ENUMERATED *, long);
+int i2d_ASN1_ENUMERATED(ASN1_ENUMERATED *, unsigned char **);
+
+ASN1_VALUE *ASN1_item_d2i(ASN1_VALUE **, const unsigned char **, long,
+                          const ASN1_ITEM *);
+int ASN1_BIT_STRING_set_bit(ASN1_BIT_STRING *, int, int);
+"""
+
+MACROS = """
+/* These became const ASN1_* in 1.1.0 */
+int i2d_ASN1_OBJECT(ASN1_OBJECT *, unsigned char **);
+int ASN1_STRING_type(ASN1_STRING *);
+int ASN1_STRING_to_UTF8(unsigned char **, ASN1_STRING *);
+long ASN1_ENUMERATED_get(ASN1_ENUMERATED *);
+int i2a_ASN1_INTEGER(BIO *, ASN1_INTEGER *);
+
+ASN1_UTF8STRING *ASN1_UTF8STRING_new(void);
+void ASN1_UTF8STRING_free(ASN1_UTF8STRING *);
+
+ASN1_BIT_STRING *ASN1_BIT_STRING_new(void);
+void ASN1_BIT_STRING_free(ASN1_BIT_STRING *);
+int i2d_ASN1_BIT_STRING(ASN1_BIT_STRING *, unsigned char **);
+int i2d_ASN1_OCTET_STRING(ASN1_OCTET_STRING *, unsigned char **);
+int i2d_ASN1_INTEGER(ASN1_INTEGER *, unsigned char **);
+/* This is not a macro, but is const on some versions of OpenSSL */
+int ASN1_BIT_STRING_get_bit(ASN1_BIT_STRING *, int);
+ASN1_TIME *M_ASN1_TIME_dup(void *);
+const ASN1_ITEM *ASN1_ITEM_ptr(ASN1_ITEM_EXP *);
+
+/* These aren't macros these arguments are all const X on openssl > 1.0.x */
+
+int ASN1_TIME_print(BIO *, ASN1_TIME *);
+int ASN1_STRING_length(ASN1_STRING *);
+ASN1_STRING *ASN1_STRING_dup(ASN1_STRING *);
+int ASN1_STRING_cmp(ASN1_STRING *, ASN1_STRING *);
+int ASN1_UTCTIME_print(BIO *, ASN1_UTCTIME *);
+
+ASN1_OCTET_STRING *ASN1_OCTET_STRING_dup(ASN1_OCTET_STRING *);
+int ASN1_OCTET_STRING_cmp(ASN1_OCTET_STRING *, ASN1_OCTET_STRING *);
+
+ASN1_INTEGER *ASN1_INTEGER_dup(ASN1_INTEGER *);
+int ASN1_INTEGER_cmp(ASN1_INTEGER *, ASN1_INTEGER *);
+long ASN1_INTEGER_get(ASN1_INTEGER *);
+
+BIGNUM *ASN1_INTEGER_to_BN(ASN1_INTEGER *, BIGNUM *);
+ASN1_INTEGER *BN_to_ASN1_INTEGER(BIGNUM *, ASN1_INTEGER *);
+
+/* These isn't a macro the arg is const on openssl 1.0.2+ */
+int ASN1_GENERALIZEDTIME_check(ASN1_GENERALIZEDTIME *);
+int ASN1_UTCTIME_check(ASN1_UTCTIME *);
+
+/* Not a macro, const on openssl 1.0 */
+int ASN1_STRING_set_default_mask_asc(char *);
+
+int i2d_ASN1_TYPE(ASN1_TYPE *, unsigned char **);
+ASN1_TYPE *d2i_ASN1_TYPE(ASN1_TYPE **, const unsigned char **, long);
+"""
+
+CUSTOMIZATIONS = """
+/* This macro is removed in 1.1.0. We re-add it if required to support
+   pyOpenSSL versions older than whatever resolves
+   https://github.com/pyca/pyopenssl/issues/431 */
+#if !defined(M_ASN1_TIME_dup)
+#define M_ASN1_TIME_dup(a) (ASN1_TIME *)ASN1_STRING_dup((const ASN1_STRING *)a)
+#endif
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/bignum.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/bignum.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/bignum.py
@@ -0,0 +1,88 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <openssl/bn.h>
+"""
+
+TYPES = """
+typedef ... BN_CTX;
+typedef ... BIGNUM;
+typedef int... BN_ULONG;
+"""
+
+FUNCTIONS = """
+BIGNUM *BN_new(void);
+void BN_free(BIGNUM *);
+
+BN_CTX *BN_CTX_new(void);
+void BN_CTX_free(BN_CTX *);
+
+void BN_CTX_start(BN_CTX *);
+BIGNUM *BN_CTX_get(BN_CTX *);
+void BN_CTX_end(BN_CTX *);
+
+BIGNUM *BN_copy(BIGNUM *, const BIGNUM *);
+BIGNUM *BN_dup(const BIGNUM *);
+
+int BN_set_word(BIGNUM *, BN_ULONG);
+BN_ULONG BN_get_word(const BIGNUM *);
+
+const BIGNUM *BN_value_one(void);
+
+char *BN_bn2hex(const BIGNUM *);
+int BN_hex2bn(BIGNUM **, const char *);
+int BN_dec2bn(BIGNUM **, const char *);
+
+int BN_bn2bin(const BIGNUM *, unsigned char *);
+BIGNUM *BN_bin2bn(const unsigned char *, int, BIGNUM *);
+
+int BN_num_bits(const BIGNUM *);
+
+int BN_cmp(const BIGNUM *, const BIGNUM *);
+int BN_add(BIGNUM *, const BIGNUM *, const BIGNUM *);
+int BN_sub(BIGNUM *, const BIGNUM *, const BIGNUM *);
+int BN_mul(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+int BN_sqr(BIGNUM *, const BIGNUM *, BN_CTX *);
+int BN_div(BIGNUM *, BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+int BN_nnmod(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+int BN_mod_add(BIGNUM *, const BIGNUM *, const BIGNUM *, const BIGNUM *,
+               BN_CTX *);
+int BN_mod_sub(BIGNUM *, const BIGNUM *, const BIGNUM *, const BIGNUM *,
+               BN_CTX *);
+int BN_mod_mul(BIGNUM *, const BIGNUM *, const BIGNUM *, const BIGNUM *,
+               BN_CTX *);
+int BN_mod_sqr(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+int BN_exp(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+int BN_mod_exp(BIGNUM *, const BIGNUM *, const BIGNUM *, const BIGNUM *,
+               BN_CTX *);
+int BN_gcd(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+BIGNUM *BN_mod_inverse(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+
+int BN_set_bit(BIGNUM *, int);
+int BN_clear_bit(BIGNUM *, int);
+
+int BN_is_bit_set(const BIGNUM *, int);
+
+int BN_mask_bits(BIGNUM *, int);
+"""
+
+MACROS = """
+int BN_num_bytes(const BIGNUM *);
+
+int BN_zero(BIGNUM *);
+int BN_one(BIGNUM *);
+int BN_mod(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+
+int BN_lshift(BIGNUM *, const BIGNUM *, int);
+int BN_lshift1(BIGNUM *, BIGNUM *);
+
+int BN_rshift(BIGNUM *, BIGNUM *, int);
+int BN_rshift1(BIGNUM *, BIGNUM *);
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/bio.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/bio.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/bio.py
@@ -0,0 +1,147 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <openssl/bio.h>
+"""
+
+TYPES = """
+typedef struct bio_st BIO;
+typedef void bio_info_cb(BIO *, int, const char *, int, long, long);
+typedef ... bio_st;
+typedef ... BIO_METHOD;
+typedef ... BUF_MEM;
+
+static const int BIO_TYPE_MEM;
+static const int BIO_TYPE_FILE;
+static const int BIO_TYPE_FD;
+static const int BIO_TYPE_SOCKET;
+static const int BIO_TYPE_CONNECT;
+static const int BIO_TYPE_ACCEPT;
+static const int BIO_TYPE_NULL;
+static const int BIO_CLOSE;
+static const int BIO_NOCLOSE;
+static const int BIO_TYPE_SOURCE_SINK;
+static const int BIO_CTRL_RESET;
+static const int BIO_CTRL_EOF;
+static const int BIO_CTRL_SET;
+static const int BIO_CTRL_SET_CLOSE;
+static const int BIO_CTRL_FLUSH;
+static const int BIO_CTRL_DUP;
+static const int BIO_CTRL_GET_CLOSE;
+static const int BIO_CTRL_INFO;
+static const int BIO_CTRL_GET;
+static const int BIO_CTRL_PENDING;
+static const int BIO_CTRL_WPENDING;
+static const int BIO_C_FILE_SEEK;
+static const int BIO_C_FILE_TELL;
+static const int BIO_TYPE_NONE;
+static const int BIO_TYPE_NBIO_TEST;
+static const int BIO_TYPE_BIO;
+static const int BIO_TYPE_DESCRIPTOR;
+static const int BIO_FLAGS_READ;
+static const int BIO_FLAGS_WRITE;
+static const int BIO_FLAGS_IO_SPECIAL;
+static const int BIO_FLAGS_RWS;
+static const int BIO_FLAGS_SHOULD_RETRY;
+static const int BIO_TYPE_NULL_FILTER;
+static const int BIO_TYPE_SSL;
+static const int BIO_TYPE_MD;
+static const int BIO_TYPE_BUFFER;
+static const int BIO_TYPE_CIPHER;
+static const int BIO_TYPE_BASE64;
+static const int BIO_TYPE_FILTER;
+"""
+
+FUNCTIONS = """
+int BIO_free(BIO *);
+void BIO_vfree(BIO *);
+void BIO_free_all(BIO *);
+BIO *BIO_push(BIO *, BIO *);
+BIO *BIO_pop(BIO *);
+BIO *BIO_next(BIO *);
+BIO *BIO_find_type(BIO *, int);
+BIO *BIO_new_file(const char *, const char *);
+BIO *BIO_new_fp(FILE *, int);
+BIO *BIO_new_fd(int, int);
+BIO *BIO_new_socket(int, int);
+long BIO_ctrl(BIO *, int, long, void *);
+long BIO_callback_ctrl(
+    BIO *,
+    int,
+    void (*)(struct bio_st *, int, const char *, int, long, long)
+);
+long BIO_int_ctrl(BIO *, int, long, int);
+size_t BIO_ctrl_pending(BIO *);
+size_t BIO_ctrl_wpending(BIO *);
+int BIO_read(BIO *, void *, int);
+int BIO_gets(BIO *, char *, int);
+int BIO_write(BIO *, const void *, int);
+int BIO_puts(BIO *, const char *);
+int BIO_method_type(const BIO *);
+"""
+
+MACROS = """
+/* Added in 1.1.0 */
+int BIO_up_ref(BIO *);
+
+/* These added const to BIO_METHOD in 1.1.0 */
+BIO *BIO_new(BIO_METHOD *);
+BIO_METHOD *BIO_s_mem(void);
+BIO_METHOD *BIO_s_file(void);
+BIO_METHOD *BIO_s_fd(void);
+BIO_METHOD *BIO_s_socket(void);
+BIO_METHOD *BIO_s_null(void);
+BIO_METHOD *BIO_f_null(void);
+BIO_METHOD *BIO_f_buffer(void);
+/* BIO_new_mem_buf became const void * in 1.0.2g */
+BIO *BIO_new_mem_buf(void *, int);
+long BIO_set_fd(BIO *, long, int);
+long BIO_get_fd(BIO *, char *);
+long BIO_set_mem_eof_return(BIO *, int);
+long BIO_get_mem_data(BIO *, char **);
+long BIO_set_mem_buf(BIO *, BUF_MEM *, int);
+long BIO_get_mem_ptr(BIO *, BUF_MEM **);
+long BIO_set_fp(BIO *, FILE *, int);
+long BIO_get_fp(BIO *, FILE **);
+long BIO_read_filename(BIO *, char *);
+long BIO_write_filename(BIO *, char *);
+long BIO_append_filename(BIO *, char *);
+long BIO_rw_filename(BIO *, char *);
+int BIO_should_read(BIO *);
+int BIO_should_write(BIO *);
+int BIO_should_io_special(BIO *);
+int BIO_retry_type(BIO *);
+int BIO_should_retry(BIO *);
+int BIO_reset(BIO *);
+int BIO_seek(BIO *, int);
+int BIO_tell(BIO *);
+int BIO_flush(BIO *);
+int BIO_eof(BIO *);
+int BIO_set_close(BIO *,long);
+int BIO_get_close(BIO *);
+int BIO_pending(BIO *);
+int BIO_wpending(BIO *);
+int BIO_get_info_callback(BIO *, bio_info_cb **);
+int BIO_set_info_callback(BIO *, bio_info_cb *);
+long BIO_get_buffer_num_lines(BIO *);
+long BIO_set_read_buffer_size(BIO *, long);
+long BIO_set_write_buffer_size(BIO *, long);
+long BIO_set_buffer_size(BIO *, long);
+long BIO_set_buffer_read_data(BIO *, void *, long);
+long BIO_set_nbio(BIO *, long);
+void BIO_set_retry_read(BIO *);
+void BIO_clear_retry_flags(BIO *);
+"""
+
+CUSTOMIZATIONS = """
+#if CRYPTOGRAPHY_OPENSSL_LESS_THAN_110PRE4 || defined(LIBRESSL_VERSION_NUMBER)
+int BIO_up_ref(BIO *b) {
+    CRYPTO_add(&b->references, 1, CRYPTO_LOCK_BIO);
+    return 1;
+}
+#endif
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/callbacks.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/callbacks.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/callbacks.py
@@ -0,0 +1,161 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+import sys
+
+import cffi
+
+INCLUDES = """
+#include <openssl/ssl.h>
+#include <openssl/x509.h>
+#include <openssl/x509_vfy.h>

From pypy.commits at gmail.com  Fri May 24 01:07:39 2019
From: pypy.commits at gmail.com (mattip)
Date: Thu, 23 May 2019 22:07:39 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: merge default into py3.6
Message-ID: <5ce77c1b.1c69fb81.951d.5b06@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r96666:6941f917df42
Date: 2019-05-24 08:05 +0300
http://bitbucket.org/pypy/pypy/changeset/6941f917df42/

Log:	merge default into py3.6

diff --git a/lib_pypy/_cffi_ssl/__init__.py b/lib_pypy/_cffi_ssl/__init__.py
new file mode 100644
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
@@ -71,6 +71,7 @@
 static const long SSL_OP_MICROSOFT_SESS_ID_BUG;
 static const long SSL_OP_NETSCAPE_CHALLENGE_BUG;
 static const long SSL_OP_NETSCAPE_REUSE_CIPHER_CHANGE_BUG;
+static const long SSL_OP_NO_SSLv2;
 static const long SSL_OP_SSLREF2_REUSE_CERT_TYPE_BUG;
 static const long SSL_OP_MICROSOFT_BIG_SSLV3_BUFFER;
 static const long SSL_OP_MSIE_SSLV2_RSA_PADDING;
diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -1,7 +1,6 @@
 import sys
 import time
-import _thread
-import socket
+import thread as _thread
 import weakref
 from _pypy_openssl import ffi
 from _pypy_openssl import lib
@@ -70,6 +69,7 @@
             globals()[name[4:]] = getattr(lib, name)
 
 OP_ALL = lib.SSL_OP_ALL & ~lib.SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS
+OP_NO_SSLv2 = lib.SSL_OP_NO_SSLv2
 
 SSL_CLIENT = 0
 SSL_SERVER = 1
@@ -78,7 +78,8 @@
 
 if lib.Cryptography_HAS_SSL2:
     PROTOCOL_SSLv2  = 0
-PROTOCOL_SSLv3  = 1
+if lib.Cryptography_HAS_SSL3_METHOD:
+    PROTOCOL_SSLv3  = 1
 PROTOCOL_SSLv23 = 2
 PROTOCOL_TLS    = PROTOCOL_SSLv23
 PROTOCOL_TLSv1    = 3
@@ -310,6 +311,9 @@
         return self.socket_type == SSL_SERVER
 
     def do_handshake(self):
+        # delay to prevent circular imports
+        import socket
+
         sock = self.get_socket_or_connection_gone()
         ssl = self.ssl
         timeout = _socket_timeout(sock)
@@ -381,6 +385,9 @@
                 return _decode_certificate(self.peer_cert)
 
     def write(self, bytestring):
+        # delay to prevent circular imports
+        import socket
+
         deadline = 0
         b = _str_to_ffi_buffer(bytestring)
         sock = self.get_socket_or_connection_gone()
@@ -439,6 +446,9 @@
             raise pyssl_error(self, length)
 
     def read(self, length, buffer_into=None):
+        # delay to prevent circular imports
+        import socket
+
         ssl = self.ssl
 
         if length < 0 and buffer_into is None:
@@ -579,6 +589,9 @@
         return sock
 
     def shutdown(self):
+        # delay to prevent circular imports
+        import socket
+
         sock = self.get_socket_or_None()
         nonblocking = False
         ssl = self.ssl
@@ -804,7 +817,7 @@
             method = lib.TLSv1_1_method()
         elif lib.Cryptography_HAS_TLSv1_2 and protocol == PROTOCOL_TLSv1_2 :
             method = lib.TLSv1_2_method()
-        elif protocol == PROTOCOL_SSLv3 and lib.Cryptography_HAS_SSL3_METHOD:
+        elif lib.Cryptography_HAS_SSL3_METHOD and protocol == PROTOCOL_SSLv3:
             method = lib.SSLv3_method()
         elif lib.Cryptography_HAS_SSL2 and protocol == PROTOCOL_SSLv2:
             method = lib.SSLv2_method()
@@ -835,7 +848,7 @@
         options = lib.SSL_OP_ALL & ~lib.SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS
         if not lib.Cryptography_HAS_SSL2 or protocol != PROTOCOL_SSLv2:
             options |= lib.SSL_OP_NO_SSLv2
-        if protocol != PROTOCOL_SSLv3:
+        if not lib.Cryptography_HAS_SSL3_METHOD or protocol != PROTOCOL_SSLv3:
             options |= lib.SSL_OP_NO_SSLv3
         # Minimal security flags for server and client side context.
         # Client sockets ignore server-side parameters.
diff --git a/lib_pypy/_cffi_ssl/_stdssl/error.py b/lib_pypy/_cffi_ssl/_stdssl/error.py
--- a/lib_pypy/_cffi_ssl/_stdssl/error.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/error.py
@@ -1,6 +1,7 @@
 import sys
 import os
 import traceback
+(??)
 from _pypy_openssl import ffi
 from _pypy_openssl import lib
 
diff --git a/lib_pypy/_cffi_ssl/_stdssl/utility.py b/lib_pypy/_cffi_ssl/_stdssl/utility.py
--- a/lib_pypy/_cffi_ssl/_stdssl/utility.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/utility.py
@@ -19,7 +19,7 @@
     elif isinstance(view, memoryview):
         # NOTE pypy limitation StringBuffer does not allow
         # to get a raw address to the string!
-        view = bytes(view)
+        view = view.tobytes()
     # dont call call ffi.from_buffer(bytes(view)), arguments
     # like ints/bools should result in a TypeError
     return ffi.from_buffer(view)
diff --git a/lib_pypy/_ssl/__init__.py b/lib_pypy/_ssl/__init__.py
--- a/lib_pypy/_ssl/__init__.py
+++ b/lib_pypy/_ssl/__init__.py
@@ -3,6 +3,8 @@
 from _cffi_ssl import _stdssl
 from _cffi_ssl._stdssl import *
 
+OP_SINGLE_DH_USE = lib.SSL_OP_SINGLE_DH_USE
+OP_SINGLE_ECDH_USE = lib.SSL_OP_SINGLE_ECDH_USE
 
 try: from __pypy__ import builtinify
 except ImportError: builtinify = lambda f: f
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -113,42 +113,6 @@
     module_suggests["cpyext"].append(("translation.shared", True))
 
 
-# NOTE: this dictionary is not used any more
-module_import_dependencies = {
-    # no _rawffi if importing rpython.rlib.clibffi raises ImportError
-    # or CompilationError or py.test.skip.Exception
-    "_rawffi"   : ["rpython.rlib.clibffi"],
-
-    "zlib"      : ["rpython.rlib.rzlib"],
-    "bz2"       : ["pypy.module.bz2.interp_bz2"],
-    "pyexpat"   : ["pypy.module.pyexpat.interp_pyexpat"],
-    "_minimal_curses": ["pypy.module._minimal_curses.fficurses"],
-    "_continuation": ["rpython.rlib.rstacklet"],
-    "_vmprof"      : ["pypy.module._vmprof.interp_vmprof"],
-    "faulthandler" : ["pypy.module._vmprof.interp_vmprof"],
-    "_lzma"     : ["pypy.module._lzma.interp_lzma"],
-    }
-
-def get_module_validator(modname):
-    # NOTE: this function is not used any more
-    if modname in module_import_dependencies:
-        modlist = module_import_dependencies[modname]
-        def validator(config):
-            from rpython.rtyper.tool.rffi_platform import CompilationError
-            try:
-                for name in modlist:
-                    __import__(name)
-            except (ImportError, CompilationError, py.test.skip.Exception) as e:
-                errcls = e.__class__.__name__
-                raise Exception(
-                    "The module %r is disabled\n" % (modname,) +
-                    "because importing %s raised %s\n" % (name, errcls) +
-                    str(e))
-        return validator
-    else:
-        return None
-
-
 pypy_optiondescription = OptionDescription("objspace", "Object Space Options", [
     OptionDescription("usemodules", "Which Modules should be used", [
         BoolOption(modname, "use module %s" % (modname, ),
@@ -157,7 +121,7 @@
                    requires=module_dependencies.get(modname, []),
                    suggests=module_suggests.get(modname, []),
                    negation=modname not in essential_modules,
-                   ) #validator=get_module_validator(modname))
+                   )
         for modname in all_modules]),
 
     BoolOption("allworkingmodules", "use as many working modules as possible",
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -29,3 +29,8 @@
 .. branch: shadowstack-issue2722
 
 Make the shadowstack size more dynamic
+
+.. branch: cffi-libs
+
+Move _ssl and _hashlib from rpython to a cffi-based module, like on python3.
+Reduces the number of problematic linked-in libraries (libssl, libcrypto)
diff --git a/pypy/tool/build_cffi_imports.py b/pypy/tool/build_cffi_imports.py
--- a/pypy/tool/build_cffi_imports.py
+++ b/pypy/tool/build_cffi_imports.py
@@ -14,7 +14,7 @@
     "tk": "_tkinter/tklib_build.py",
     "curses": "_curses_build.py" if sys.platform != "win32" else None,
     "syslog": "_syslog_build.py" if sys.platform != "win32" else None,
-    "_gdbm": "_gdbm_build.py"  if sys.platform != "win32" else None,
+    "gdbm": "_gdbm_build.py"  if sys.platform != "win32" else None,
     "pwdgrp": "_pwdgrp_build.py" if sys.platform != "win32" else None,
     "resource": "_resource_build.py" if sys.platform != "win32" else None,
     "lzma": "_lzma_build.py",

From pypy.commits at gmail.com  Fri May 24 03:06:28 2019
From: pypy.commits at gmail.com (mattip)
Date: Fri, 24 May 2019 00:06:28 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: fix merge
Message-ID: <5ce797f4.1c69fb81.a156f.5f2b@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r96667:385db1f245d0
Date: 2019-05-24 10:05 +0300
http://bitbucket.org/pypy/pypy/changeset/385db1f245d0/

Log:	fix merge

diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -1,6 +1,6 @@
 import sys
 import time
-import thread as _thread
+import _thread
 import weakref
 from _pypy_openssl import ffi
 from _pypy_openssl import lib
diff --git a/lib_pypy/_cffi_ssl/_stdssl/error.py b/lib_pypy/_cffi_ssl/_stdssl/error.py
--- a/lib_pypy/_cffi_ssl/_stdssl/error.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/error.py
@@ -1,7 +1,6 @@
 import sys
 import os
 import traceback
-(??)
 from _pypy_openssl import ffi
 from _pypy_openssl import lib
 

From pypy.commits at gmail.com  Fri May 24 13:00:28 2019
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 24 May 2019 10:00:28 -0700 (PDT)
Subject: [pypy-commit] pypy optimizeopt-cleanup: small cleanup
Message-ID: <5ce8232c.1c69fb81.d521e.b772@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: optimizeopt-cleanup
Changeset: r96668:e246a0d898ed
Date: 2019-05-24 17:54 +0100
http://bitbucket.org/pypy/pypy/changeset/e246a0d898ed/

Log:	small cleanup

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -379,8 +379,6 @@
             return None
 
     label_op = loop_info.label_op
-    if label_op is None:
-        assert False, "unreachable code" # hint for some strange tests
     label_token = label_op.getdescr()
     assert isinstance(label_token, TargetToken)
     if label_token.short_preamble:
@@ -388,9 +386,8 @@
             label_token.short_preamble, metainterp.box_names_memo)
     loop = partial_trace
     loop.original_jitcell_token = loop_jitcell_token
-    loop.operations = (loop.operations + loop_info.extra_same_as +
-                       [loop_info.label_op]
-                       + loop_ops)
+    loop.operations = (
+        loop.operations + loop_info.extra_same_as + [label_op] + loop_ops)
 
     quasi_immutable_deps = {}
     if loop_info.quasi_immutable_deps:

From pypy.commits at gmail.com  Fri May 24 16:31:11 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 24 May 2019 13:31:11 -0700 (PDT)
Subject: [pypy-commit] pypy fix-vmprof-memory-tracking: branch to fix the
 broken memory-tracking on PyPy
Message-ID: <5ce8548f.1c69fb81.7c782.ccde@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: fix-vmprof-memory-tracking
Changeset: r96669:36949270bede
Date: 2019-05-24 17:04 +0200
http://bitbucket.org/pypy/pypy/changeset/36949270bede/

Log:	branch to fix the broken memory-tracking on PyPy


From pypy.commits at gmail.com  Fri May 24 16:31:14 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 24 May 2019 13:31:14 -0700 (PDT)
Subject: [pypy-commit] pypy fix-vmprof-memory-tracking: aaargh,
 the two arguments to vmprof_init were swapped, breaking memory
Message-ID: <5ce85492.1c69fb81.d9af7.e500@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: fix-vmprof-memory-tracking
Changeset: r96670:560b75269a02
Date: 2019-05-24 17:37 +0200
http://bitbucket.org/pypy/pypy/changeset/560b75269a02/

Log:	aaargh, the two arguments to vmprof_init were swapped, breaking
	memory profiling on PyPy for non-deep reasons.

diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -143,7 +143,7 @@
             native = 0 # force disabled on Windows
         lines = 0 # not supported on PyPy currently
 
-        p_error = self.cintf.vmprof_init(fileno, interval, lines, memory, "pypy", native, real_time)
+        p_error = self.cintf.vmprof_init(fileno, interval, memory, lines, "pypy", native, real_time)
         if p_error:
             raise VMProfError(rffi.charp2str(p_error))
 

From pypy.commits at gmail.com  Fri May 24 16:31:17 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 24 May 2019 13:31:17 -0700 (PDT)
Subject: [pypy-commit] pypy fix-vmprof-memory-tracking: remove commented out
 code, this just works now
Message-ID: <5ce85495.1c69fb81.18154.d864@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: fix-vmprof-memory-tracking
Changeset: r96671:40f08696399c
Date: 2019-05-24 17:37 +0200
http://bitbucket.org/pypy/pypy/changeset/40f08696399c/

Log:	remove commented out code, this just works now

diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -60,11 +60,6 @@
     'interval' is a float representing the sampling interval, in seconds.
     Must be smaller than 1.0
     """
-    w_modules = space.sys.get('modules')
-    #if space.contains_w(w_modules, space.newtext('_continuation')):
-    #    space.warn(space.newtext("Using _continuation/greenlet/stacklet together "
-    #                             "with vmprof will crash"),
-    #               space.w_RuntimeWarning)
     try:
         rvmprof.enable(fileno, period, memory, native, real_time)
     except rvmprof.VMProfError as e:

From pypy.commits at gmail.com  Fri May 24 16:52:41 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 24 May 2019 13:52:41 -0700 (PDT)
Subject: [pypy-commit] pypy fix-vmprof-memory-tracking: add test that would
 have shown the problem (why was it not there in the first place?)
Message-ID: <5ce85999.1c69fb81.8758a.cfe0@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: fix-vmprof-memory-tracking
Changeset: r96672:bb3438f783d5
Date: 2019-05-24 22:52 +0200
http://bitbucket.org/pypy/pypy/changeset/bb3438f783d5/

Log:	add test that would have shown the problem (why was it not there in
	the first place?)

diff --git a/rpython/rlib/rvmprof/test/test_rvmprof.py b/rpython/rlib/rvmprof/test/test_rvmprof.py
--- a/rpython/rlib/rvmprof/test/test_rvmprof.py
+++ b/rpython/rlib/rvmprof/test/test_rvmprof.py
@@ -98,12 +98,12 @@
         self.tmpfilename = str(self.tmpfile)
         super(RVMProfSamplingTest, self).init()
 
-    ENTRY_POINT_ARGS = (int, float)
-    def entry_point(self, value, delta_t):
+    ENTRY_POINT_ARGS = (int, float, int)
+    def entry_point(self, value, delta_t, memory=0):
         code = self.MyCode('py:code:52:test_enable')
         rvmprof.register_code(code, self.MyCode.get_name)
         fd = os.open(self.tmpfilename, os.O_WRONLY | os.O_CREAT, 0666)
-        rvmprof.enable(fd, self.SAMPLING_INTERVAL)
+        rvmprof.enable(fd, self.SAMPLING_INTERVAL, memory=memory)
         start = time.time()
         res = 0
         while time.time() < start+delta_t:
@@ -128,17 +128,25 @@
 
     def test(self):
         from vmprof import read_profile
-        assert self.entry_point(10**4, 0.1) == 99990000
+        assert self.entry_point(10**4, 0.1, 0) == 99990000
         assert self.tmpfile.check()
         self.tmpfile.remove()
         #
-        assert self.rpy_entry_point(10**4, 0.5) == 99990000
+        assert self.rpy_entry_point(10**4, 0.5, 0) == 99990000
         assert self.tmpfile.check()
         prof = read_profile(self.tmpfilename)
         tree = prof.get_tree()
         assert tree.name == 'py:code:52:test_enable'
         assert self.approx_equal(tree.count, 0.5/self.SAMPLING_INTERVAL)
 
+    def test_mem(self):
+        from vmprof import read_profile
+        assert self.rpy_entry_point(10**4, 0.5, 1) == 99990000
+        assert self.tmpfile.check()
+        prof = read_profile(self.tmpfilename)
+        assert prof.profile_memory
+        assert all(p[-1] > 0 for p in prof.profiles)
+
 
 class TestNative(RVMProfSamplingTest):
 
@@ -177,7 +185,7 @@
     def test(self):
         from vmprof import read_profile
         # from vmprof.show import PrettyPrinter
-        assert self.rpy_entry_point(3, 0.5) == 42000
+        assert self.rpy_entry_point(3, 0.5, 0) == 42000
         assert self.tmpfile.check()
 
         prof = read_profile(self.tmpfilename)

From pypy.commits at gmail.com  Sat May 25 02:08:47 2019
From: pypy.commits at gmail.com (mattip)
Date: Fri, 24 May 2019 23:08:47 -0700 (PDT)
Subject: [pypy-commit] pypy default: untwist circular imports by adding
 legacy attributes to socket after import
Message-ID: <5ce8dbef.1c69fb81.69a39.4a94@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r96673:34c2a20bdb33
Date: 2019-05-25 09:07 +0300
http://bitbucket.org/pypy/pypy/changeset/34c2a20bdb33/

Log:	untwist circular imports by adding legacy attributes to socket after
	import

diff --git a/lib-python/2.7/socket.py b/lib-python/2.7/socket.py
--- a/lib-python/2.7/socket.py
+++ b/lib-python/2.7/socket.py
@@ -61,20 +61,22 @@
                       DeprecationWarning, stacklevel=2)
         return _realssl.sslwrap_simple(sock, keyfile, certfile)
 
-    # we need to import the same constants we used to...
-    from _ssl import SSLError as sslerror
-    from _ssl import \
-         RAND_add, \
-         RAND_status, \
-         SSL_ERROR_ZERO_RETURN, \
-         SSL_ERROR_WANT_READ, \
-         SSL_ERROR_WANT_WRITE, \
-         SSL_ERROR_WANT_X509_LOOKUP, \
-         SSL_ERROR_SYSCALL, \
-         SSL_ERROR_SSL, \
-         SSL_ERROR_WANT_CONNECT, \
-         SSL_ERROR_EOF, \
-         SSL_ERROR_INVALID_ERROR_CODE
+    # we need to import the same constants we used to, 
+    # see lib_pypy/_cffi_ssl/_stdssl/error.py and __init__.py to prevent
+    # circular import
+    # from _ssl import SSLError as sslerror
+    # from _ssl import \
+         # RAND_add, \
+         # RAND_status
+         # SSL_ERROR_ZERO_RETURN, \
+         # SSL_ERROR_WANT_READ, \
+         # SSL_ERROR_WANT_WRITE, \
+         # SSL_ERROR_WANT_X509_LOOKUP, \
+         # SSL_ERROR_SYSCALL, \
+         # SSL_ERROR_SSL, \
+         # SSL_ERROR_WANT_CONNECT, \
+         # SSL_ERROR_EOF, \
+         # SSL_ERROR_INVALID_ERROR_CODE
     try:
         from _ssl import RAND_egd
     except ImportError:
diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -20,6 +20,7 @@
         pyerr_write_unraisable)
 from _cffi_ssl._stdssl import error
 from select import select
+import socket
 
 if sys.platform == 'win32':
     from _cffi_ssl._stdssl.win32_extra import enum_certificates, enum_crls
@@ -306,9 +307,6 @@
         return self.socket_type == SSL_SERVER
 
     def do_handshake(self):
-        # delay to prevent circular imports
-        import socket
-
         sock = self.get_socket_or_connection_gone()
         ssl = self.ssl
         timeout = _socket_timeout(sock)
@@ -372,9 +370,6 @@
                 return _decode_certificate(self.peer_cert)
 
     def write(self, bytestring):
-        # delay to prevent circular imports
-        import socket
-
         deadline = 0
         b = _str_to_ffi_buffer(bytestring)
         sock = self.get_socket_or_connection_gone()
@@ -425,9 +420,6 @@
             raise pyssl_error(self, length)
 
     def read(self, length, buffer_into=None):
-        # delay to prevent circular imports
-        import socket
-
         ssl = self.ssl
 
         if length < 0 and buffer_into is None:
@@ -561,9 +553,6 @@
         return sock
 
     def shutdown(self):
-        # delay to prevent circular imports
-        import socket
-
         sock = self.get_socket_or_None()
         nonblocking = False
         ssl = self.ssl
@@ -1545,3 +1534,5 @@
                            "enough data to seed the PRNG");
         return bytecount
 
+socket.RAND_add = RAND_add
+socket.RAND_status = RAND_status
diff --git a/lib_pypy/_cffi_ssl/_stdssl/error.py b/lib_pypy/_cffi_ssl/_stdssl/error.py
--- a/lib_pypy/_cffi_ssl/_stdssl/error.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/error.py
@@ -27,6 +27,13 @@
         if self.strerror and isinstance(self.strerror, str):
             return self.strerror
         return str(self.args)
+# these are expected on socket as well
+socket.sslerror = SSLError
+for v in [ 'SSL_ERROR_ZERO_RETURN', 'SSL_ERROR_WANT_READ',
+     'SSL_ERROR_WANT_WRITE', 'SSL_ERROR_WANT_X509_LOOKUP', 'SSL_ERROR_SYSCALL',
+     'SSL_ERROR_SSL', 'SSL_ERROR_WANT_CONNECT', 'SSL_ERROR_EOF',
+     'SSL_ERROR_INVALID_ERROR_CODE' ]:
+    setattr(socket, v, locals()[v]) 
 
 class SSLZeroReturnError(SSLError):
     """ SSL/TLS session closed cleanly. """

From pypy.commits at gmail.com  Sat May 25 02:49:36 2019
From: pypy.commits at gmail.com (mattip)
Date: Fri, 24 May 2019 23:49:36 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix more tests
Message-ID: <5ce8e580.1c69fb81.e9a02.0549@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r96674:48ae5caa94b0
Date: 2019-05-25 09:45 +0300
http://bitbucket.org/pypy/pypy/changeset/48ae5caa94b0/

Log:	fix more tests

diff --git a/lib_pypy/_hashlib/__init__.py b/lib_pypy/_hashlib/__init__.py
--- a/lib_pypy/_hashlib/__init__.py
+++ b/lib_pypy/_hashlib/__init__.py
@@ -57,7 +57,7 @@
 
     def update(self, string):
         if isinstance(string, unicode):
-            buf = ffi.from_buffer(string.encode('utf-8'))
+            buf = ffi.from_buffer(string.encode('ascii'))
         else:
             buf = ffi.from_buffer(string)
         with self.lock:
diff --git a/pypy/module/_md5/test/test_md5.py b/pypy/module/_md5/test/test_md5.py
--- a/pypy/module/_md5/test/test_md5.py
+++ b/pypy/module/_md5/test/test_md5.py
@@ -24,7 +24,8 @@
         assert self.md5.md5().digest_size == 16
         if sys.version_info >= (2, 5):
             assert self.md5.blocksize == 1
-            assert self.md5.md5().digestsize == 16
+            # implementation detail, not part of the API
+            # assert self.md5.md5().digestsize == 16
 
     def test_MD5Type(self):
         """
diff --git a/pypy/module/_sha/test/test_sha.py b/pypy/module/_sha/test/test_sha.py
--- a/pypy/module/_sha/test/test_sha.py
+++ b/pypy/module/_sha/test/test_sha.py
@@ -23,7 +23,8 @@
         assert self.sha.digestsize == 20
         d = self.sha.sha()
         assert d.digest_size == 20
-        assert d.digestsize == 20
+        # implementation detail, not part of the API
+        # assert d.digestsize == 20
 
     def test_SHAType(self):
         """

From pypy.commits at gmail.com  Sat May 25 05:40:30 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Sat, 25 May 2019 02:40:30 -0700 (PDT)
Subject: [pypy-commit] pypy default: stop running this record_exact_class
 test in the backends. it's failing for
Message-ID: <5ce90d8e.1c69fb81.bdf96.2707@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r96675:f48749310466
Date: 2019-05-25 00:33 +0200
http://bitbucket.org/pypy/pypy/changeset/f48749310466/

Log:	stop running this record_exact_class test in the backends. it's
	failing for unrelated reasons, and it's a frontend test anywa test
	in the backends.

diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -3989,55 +3989,6 @@
         # here it works again
         self.check_operations_history(guard_class=0, record_exact_class=1)
 
-    def test_record_exact_class_nonconst(self):
-        class Base(object):
-            def f(self):
-                raise NotImplementedError
-            def g(self):
-                raise NotImplementedError
-        class A(Base):
-            def f(self):
-                return self.a
-            def g(self):
-                return self.a + 1
-        class B(Base):
-            def f(self):
-                return self.b
-            def g(self):
-                return self.b + 1
-        class C(B):
-            def f(self):
-                self.c += 1
-                return self.c
-            def g(self):
-                return self.c + 1
-        @dont_look_inside
-        def make(x):
-            if x > 0:
-                a = A()
-                a.a = x + 1
-            elif x < 0:
-                a = B()
-                a.b = -x
-            else:
-                a = C()
-                a.c = 10
-            return a, type(a)
-        def f(x):
-            a, cls = make(x)
-            record_exact_class(a, cls)
-            if x > 0:
-                z = a.f()
-            elif x < 0:
-                z = a.f()
-            else:
-                z = a.f()
-            return z + a.g()
-        res1 = f(6)
-        res2 = self.interp_operations(f, [6])
-        assert res1 == res2
-        self.check_operations_history(guard_class=1, record_exact_class=0)
-
     def test_generator(self):
         def g(n):
             yield n+1
@@ -4889,3 +4840,52 @@
 
         res = self.meta_interp(f, [0])
         assert res == f(0)
+
+    def test_record_exact_class_nonconst(self):
+        class Base(object):
+            def f(self):
+                raise NotImplementedError
+            def g(self):
+                raise NotImplementedError
+        class A(Base):
+            def f(self):
+                return self.a
+            def g(self):
+                return self.a + 1
+        class B(Base):
+            def f(self):
+                return self.b
+            def g(self):
+                return self.b + 1
+        class C(B):
+            def f(self):
+                self.c += 1
+                return self.c
+            def g(self):
+                return self.c + 1
+        @dont_look_inside
+        def make(x):
+            if x > 0:
+                a = A()
+                a.a = x + 1
+            elif x < 0:
+                a = B()
+                a.b = -x
+            else:
+                a = C()
+                a.c = 10
+            return a, type(a)
+        def f(x):
+            a, cls = make(x)
+            record_exact_class(a, cls)
+            if x > 0:
+                z = a.f()
+            elif x < 0:
+                z = a.f()
+            else:
+                z = a.f()
+            return z + a.g()
+        res1 = f(6)
+        res2 = self.interp_operations(f, [6])
+        assert res1 == res2
+        self.check_operations_history(guard_class=1, record_exact_class=0)

From pypy.commits at gmail.com  Sat May 25 05:40:32 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Sat, 25 May 2019 02:40:32 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge fix-vmprof-memory-tracking
Message-ID: <5ce90d90.1c69fb81.10524.dfba@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r96676:9cfdc2c91a37
Date: 2019-05-25 11:27 +0200
http://bitbucket.org/pypy/pypy/changeset/9cfdc2c91a37/

Log:	merge fix-vmprof-memory-tracking

diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -60,11 +60,6 @@
     'interval' is a float representing the sampling interval, in seconds.
     Must be smaller than 1.0
     """
-    w_modules = space.sys.get('modules')
-    #if space.contains_w(w_modules, space.newtext('_continuation')):
-    #    space.warn(space.newtext("Using _continuation/greenlet/stacklet together "
-    #                             "with vmprof will crash"),
-    #               space.w_RuntimeWarning)
     try:
         rvmprof.enable(fileno, period, memory, native, real_time)
     except rvmprof.VMProfError as e:
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -143,7 +143,7 @@
             native = 0 # force disabled on Windows
         lines = 0 # not supported on PyPy currently
 
-        p_error = self.cintf.vmprof_init(fileno, interval, lines, memory, "pypy", native, real_time)
+        p_error = self.cintf.vmprof_init(fileno, interval, memory, lines, "pypy", native, real_time)
         if p_error:
             raise VMProfError(rffi.charp2str(p_error))
 
diff --git a/rpython/rlib/rvmprof/test/test_rvmprof.py b/rpython/rlib/rvmprof/test/test_rvmprof.py
--- a/rpython/rlib/rvmprof/test/test_rvmprof.py
+++ b/rpython/rlib/rvmprof/test/test_rvmprof.py
@@ -98,12 +98,12 @@
         self.tmpfilename = str(self.tmpfile)
         super(RVMProfSamplingTest, self).init()
 
-    ENTRY_POINT_ARGS = (int, float)
-    def entry_point(self, value, delta_t):
+    ENTRY_POINT_ARGS = (int, float, int)
+    def entry_point(self, value, delta_t, memory=0):
         code = self.MyCode('py:code:52:test_enable')
         rvmprof.register_code(code, self.MyCode.get_name)
         fd = os.open(self.tmpfilename, os.O_WRONLY | os.O_CREAT, 0666)
-        rvmprof.enable(fd, self.SAMPLING_INTERVAL)
+        rvmprof.enable(fd, self.SAMPLING_INTERVAL, memory=memory)
         start = time.time()
         res = 0
         while time.time() < start+delta_t:
@@ -128,17 +128,25 @@
 
     def test(self):
         from vmprof import read_profile
-        assert self.entry_point(10**4, 0.1) == 99990000
+        assert self.entry_point(10**4, 0.1, 0) == 99990000
         assert self.tmpfile.check()
         self.tmpfile.remove()
         #
-        assert self.rpy_entry_point(10**4, 0.5) == 99990000
+        assert self.rpy_entry_point(10**4, 0.5, 0) == 99990000
         assert self.tmpfile.check()
         prof = read_profile(self.tmpfilename)
         tree = prof.get_tree()
         assert tree.name == 'py:code:52:test_enable'
         assert self.approx_equal(tree.count, 0.5/self.SAMPLING_INTERVAL)
 
+    def test_mem(self):
+        from vmprof import read_profile
+        assert self.rpy_entry_point(10**4, 0.5, 1) == 99990000
+        assert self.tmpfile.check()
+        prof = read_profile(self.tmpfilename)
+        assert prof.profile_memory
+        assert all(p[-1] > 0 for p in prof.profiles)
+
 
 class TestNative(RVMProfSamplingTest):
 
@@ -177,7 +185,7 @@
     def test(self):
         from vmprof import read_profile
         # from vmprof.show import PrettyPrinter
-        assert self.rpy_entry_point(3, 0.5) == 42000
+        assert self.rpy_entry_point(3, 0.5, 0) == 42000
         assert self.tmpfile.check()
 
         prof = read_profile(self.tmpfilename)

From pypy.commits at gmail.com  Sat May 25 05:40:34 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Sat, 25 May 2019 02:40:34 -0700 (PDT)
Subject: [pypy-commit] pypy default: document merged branch
Message-ID: <5ce90d92.1c69fb81.22ab3.8ee7@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r96677:911de07a90eb
Date: 2019-05-25 11:29 +0200
http://bitbucket.org/pypy/pypy/changeset/911de07a90eb/

Log:	document merged branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -30,3 +30,7 @@
 
 Move _ssl and _hashlib from rpython to a cffi-based module, like on python3.
 Reduces the number of problematic linked-in libraries (libssl, libcrypto)
+
+.. branch: fix-vmprof-memory-tracking
+
+Fix a bug that prevent memory-tracking in vmprof working on PyPy.

From pypy.commits at gmail.com  Sat May 25 05:40:36 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Sat, 25 May 2019 02:40:36 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <5ce90d94.1c69fb81.69a39.64c5@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r96678:aeb506f08f30
Date: 2019-05-25 11:39 +0200
http://bitbucket.org/pypy/pypy/changeset/aeb506f08f30/

Log:	merge heads

diff --git a/lib-python/2.7/socket.py b/lib-python/2.7/socket.py
--- a/lib-python/2.7/socket.py
+++ b/lib-python/2.7/socket.py
@@ -61,20 +61,22 @@
                       DeprecationWarning, stacklevel=2)
         return _realssl.sslwrap_simple(sock, keyfile, certfile)
 
-    # we need to import the same constants we used to...
-    from _ssl import SSLError as sslerror
-    from _ssl import \
-         RAND_add, \
-         RAND_status, \
-         SSL_ERROR_ZERO_RETURN, \
-         SSL_ERROR_WANT_READ, \
-         SSL_ERROR_WANT_WRITE, \
-         SSL_ERROR_WANT_X509_LOOKUP, \
-         SSL_ERROR_SYSCALL, \
-         SSL_ERROR_SSL, \
-         SSL_ERROR_WANT_CONNECT, \
-         SSL_ERROR_EOF, \
-         SSL_ERROR_INVALID_ERROR_CODE
+    # we need to import the same constants we used to, 
+    # see lib_pypy/_cffi_ssl/_stdssl/error.py and __init__.py to prevent
+    # circular import
+    # from _ssl import SSLError as sslerror
+    # from _ssl import \
+         # RAND_add, \
+         # RAND_status
+         # SSL_ERROR_ZERO_RETURN, \
+         # SSL_ERROR_WANT_READ, \
+         # SSL_ERROR_WANT_WRITE, \
+         # SSL_ERROR_WANT_X509_LOOKUP, \
+         # SSL_ERROR_SYSCALL, \
+         # SSL_ERROR_SSL, \
+         # SSL_ERROR_WANT_CONNECT, \
+         # SSL_ERROR_EOF, \
+         # SSL_ERROR_INVALID_ERROR_CODE
     try:
         from _ssl import RAND_egd
     except ImportError:
diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -20,6 +20,7 @@
         pyerr_write_unraisable)
 from _cffi_ssl._stdssl import error
 from select import select
+import socket
 
 if sys.platform == 'win32':
     from _cffi_ssl._stdssl.win32_extra import enum_certificates, enum_crls
@@ -306,9 +307,6 @@
         return self.socket_type == SSL_SERVER
 
     def do_handshake(self):
-        # delay to prevent circular imports
-        import socket
-
         sock = self.get_socket_or_connection_gone()
         ssl = self.ssl
         timeout = _socket_timeout(sock)
@@ -372,9 +370,6 @@
                 return _decode_certificate(self.peer_cert)
 
     def write(self, bytestring):
-        # delay to prevent circular imports
-        import socket
-
         deadline = 0
         b = _str_to_ffi_buffer(bytestring)
         sock = self.get_socket_or_connection_gone()
@@ -425,9 +420,6 @@
             raise pyssl_error(self, length)
 
     def read(self, length, buffer_into=None):
-        # delay to prevent circular imports
-        import socket
-
         ssl = self.ssl
 
         if length < 0 and buffer_into is None:
@@ -561,9 +553,6 @@
         return sock
 
     def shutdown(self):
-        # delay to prevent circular imports
-        import socket
-
         sock = self.get_socket_or_None()
         nonblocking = False
         ssl = self.ssl
@@ -1545,3 +1534,5 @@
                            "enough data to seed the PRNG");
         return bytecount
 
+socket.RAND_add = RAND_add
+socket.RAND_status = RAND_status
diff --git a/lib_pypy/_cffi_ssl/_stdssl/error.py b/lib_pypy/_cffi_ssl/_stdssl/error.py
--- a/lib_pypy/_cffi_ssl/_stdssl/error.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/error.py
@@ -27,6 +27,13 @@
         if self.strerror and isinstance(self.strerror, str):
             return self.strerror
         return str(self.args)
+# these are expected on socket as well
+socket.sslerror = SSLError
+for v in [ 'SSL_ERROR_ZERO_RETURN', 'SSL_ERROR_WANT_READ',
+     'SSL_ERROR_WANT_WRITE', 'SSL_ERROR_WANT_X509_LOOKUP', 'SSL_ERROR_SYSCALL',
+     'SSL_ERROR_SSL', 'SSL_ERROR_WANT_CONNECT', 'SSL_ERROR_EOF',
+     'SSL_ERROR_INVALID_ERROR_CODE' ]:
+    setattr(socket, v, locals()[v]) 
 
 class SSLZeroReturnError(SSLError):
     """ SSL/TLS session closed cleanly. """
diff --git a/lib_pypy/_hashlib/__init__.py b/lib_pypy/_hashlib/__init__.py
--- a/lib_pypy/_hashlib/__init__.py
+++ b/lib_pypy/_hashlib/__init__.py
@@ -57,7 +57,7 @@
 
     def update(self, string):
         if isinstance(string, unicode):
-            buf = ffi.from_buffer(string.encode('utf-8'))
+            buf = ffi.from_buffer(string.encode('ascii'))
         else:
             buf = ffi.from_buffer(string)
         with self.lock:
diff --git a/pypy/module/_md5/test/test_md5.py b/pypy/module/_md5/test/test_md5.py
--- a/pypy/module/_md5/test/test_md5.py
+++ b/pypy/module/_md5/test/test_md5.py
@@ -24,7 +24,8 @@
         assert self.md5.md5().digest_size == 16
         if sys.version_info >= (2, 5):
             assert self.md5.blocksize == 1
-            assert self.md5.md5().digestsize == 16
+            # implementation detail, not part of the API
+            # assert self.md5.md5().digestsize == 16
 
     def test_MD5Type(self):
         """
diff --git a/pypy/module/_sha/test/test_sha.py b/pypy/module/_sha/test/test_sha.py
--- a/pypy/module/_sha/test/test_sha.py
+++ b/pypy/module/_sha/test/test_sha.py
@@ -23,7 +23,8 @@
         assert self.sha.digestsize == 20
         d = self.sha.sha()
         assert d.digest_size == 20
-        assert d.digestsize == 20
+        # implementation detail, not part of the API
+        # assert d.digestsize == 20
 
     def test_SHAType(self):
         """

From pypy.commits at gmail.com  Sat May 25 08:48:37 2019
From: pypy.commits at gmail.com (mattip)
Date: Sat, 25 May 2019 05:48:37 -0700 (PDT)
Subject: [pypy-commit] pypy default: unskip tests,
 fix a failure. Be careful when merging to py3.6!
Message-ID: <5ce939a5.1c69fb81.16d3a.6580@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r96679:86a195a12002
Date: 2019-05-25 15:47 +0300
http://bitbucket.org/pypy/pypy/changeset/86a195a12002/

Log:	unskip tests, fix a failure. Be careful when merging to py3.6!

diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -401,7 +401,7 @@
     RegrTest('test_source_encoding.py'),
     RegrTest('test_spwd.py'),
     RegrTest('test_sqlite.py', usemodules="thread _rawffi zlib"),
-    RegrTest('test_ssl.py', usemodules='_ssl _socket select'),
+    RegrTest('test_ssl.py', usemodules='_socket select'),
     RegrTest('test_startfile.py'),
     RegrTest('test_stat.py'),
     RegrTest('test_str.py', core=True),
diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -1025,7 +1025,7 @@
             self._add_ca_certs(buf, len(buf), ca_file_type)
 
         # load cafile or capath
-        if cafile or capath:
+        if cafile is not None or capath is not None:
             if cafile is None:
                 cafilebuf = ffi.NULL
             else:

From pypy.commits at gmail.com  Sat May 25 14:52:48 2019
From: pypy.commits at gmail.com (mattip)
Date: Sat, 25 May 2019 11:52:48 -0700 (PDT)
Subject: [pypy-commit] pypy stdlib-2.7.16: update micro version
Message-ID: <5ce98f00.1c69fb81.8758a.9d66@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: stdlib-2.7.16
Changeset: r96681:8931263cc7ac
Date: 2019-05-25 21:52 +0300
http://bitbucket.org/pypy/pypy/changeset/8931263cc7ac/

Log:	update micro version

diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -21,7 +21,7 @@
 /* Version parsed out into numeric values */
 #define PY_MAJOR_VERSION	2
 #define PY_MINOR_VERSION	7
-#define PY_MICRO_VERSION	13
+#define PY_MICRO_VERSION	16
 #define PY_RELEASE_LEVEL	PY_RELEASE_LEVEL_FINAL
 #define PY_RELEASE_SERIAL	0
 
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -6,7 +6,7 @@
 from pypy.interpreter import gateway
 
 #XXX # the release serial 42 is not in range(16)
-CPYTHON_VERSION            = (2, 7, 13, "final", 42)
+CPYTHON_VERSION            = (2, 7, 16, "final", 42)
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 

From pypy.commits at gmail.com  Sat May 25 14:52:46 2019
From: pypy.commits at gmail.com (mattip)
Date: Sat, 25 May 2019 11:52:46 -0700 (PDT)
Subject: [pypy-commit] pypy stdlib-2.7.16: merge default into branch
Message-ID: <5ce98efe.1c69fb81.8469f.d675@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: stdlib-2.7.16
Changeset: r96680:51ed8b3f8235
Date: 2019-05-25 21:46 +0300
http://bitbucket.org/pypy/pypy/changeset/51ed8b3f8235/

Log:	merge default into branch

diff too long, truncating to 2000 out of 12085 lines

diff --git a/lib-python/2.7/socket.py b/lib-python/2.7/socket.py
--- a/lib-python/2.7/socket.py
+++ b/lib-python/2.7/socket.py
@@ -61,20 +61,22 @@
                       DeprecationWarning, stacklevel=2)
         return _realssl.sslwrap_simple(sock, keyfile, certfile)
 
-    # we need to import the same constants we used to...
-    from _ssl import SSLError as sslerror
-    from _ssl import \
-         RAND_add, \
-         RAND_status, \
-         SSL_ERROR_ZERO_RETURN, \
-         SSL_ERROR_WANT_READ, \
-         SSL_ERROR_WANT_WRITE, \
-         SSL_ERROR_WANT_X509_LOOKUP, \
-         SSL_ERROR_SYSCALL, \
-         SSL_ERROR_SSL, \
-         SSL_ERROR_WANT_CONNECT, \
-         SSL_ERROR_EOF, \
-         SSL_ERROR_INVALID_ERROR_CODE
+    # we need to import the same constants we used to, 
+    # see lib_pypy/_cffi_ssl/_stdssl/error.py and __init__.py to prevent
+    # circular import
+    # from _ssl import SSLError as sslerror
+    # from _ssl import \
+         # RAND_add, \
+         # RAND_status
+         # SSL_ERROR_ZERO_RETURN, \
+         # SSL_ERROR_WANT_READ, \
+         # SSL_ERROR_WANT_WRITE, \
+         # SSL_ERROR_WANT_X509_LOOKUP, \
+         # SSL_ERROR_SYSCALL, \
+         # SSL_ERROR_SSL, \
+         # SSL_ERROR_WANT_CONNECT, \
+         # SSL_ERROR_EOF, \
+         # SSL_ERROR_INVALID_ERROR_CODE
     try:
         from _ssl import RAND_egd
     except ImportError:
diff --git a/lib-python/2.7/test/capath/efa5f9c3.0 b/lib-python/2.7/test/capath/efa5f9c3.0
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/capath/efa5f9c3.0
@@ -0,0 +1,34 @@
+-----BEGIN CERTIFICATE-----
+MIIF9zCCA9+gAwIBAgIUH98b4Fw/DyugC9cV7VK7ZODzHsIwDQYJKoZIhvcNAQEL
+BQAwgYoxCzAJBgNVBAYTAlhZMRcwFQYDVQQIDA5DYXN0bGUgQW50aHJheDEYMBYG
+A1UEBwwPQXJndW1lbnQgQ2xpbmljMSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUg
+Rm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0aG9udGVzdC5uZXQw
+HhcNMTkwNTA4MDEwMjQzWhcNMjcwNzI0MDEwMjQzWjCBijELMAkGA1UEBhMCWFkx
+FzAVBgNVBAgMDkNhc3RsZSBBbnRocmF4MRgwFgYDVQQHDA9Bcmd1bWVudCBDbGlu
+aWMxIzAhBgNVBAoMGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMSMwIQYDVQQD
+DBpzZWxmLXNpZ25lZC5weXRob250ZXN0Lm5ldDCCAiIwDQYJKoZIhvcNAQEBBQAD
+ggIPADCCAgoCggIBAMKdJlyCThkahwoBb7pl5q64Pe9Fn5jrIvzsveHTc97TpjV2
+RLfICnXKrltPk/ohkVl6K5SUZQZwMVzFubkyxE0nZPHYHlpiKWQxbsYVkYv01rix
+IFdLvaxxbGYke2jwQao31s4o61AdlsfK1SdpHQUynBBMssqI3SB4XPmcA7e+wEEx
+jxjVish4ixA1vuIZOx8yibu+CFCf/geEjoBMF3QPdzULzlrCSw8k/45iZCSoNbvK
+DoL4TVV07PHOxpheDh8ZQmepGvU6pVqhb9m4lgmV0OGWHgozd5Ur9CbTVDmxIEz3
+TSoRtNJK7qtyZdGNqwjksQxgZTjM/d/Lm/BJG99AiOmYOjsl9gbQMZgvQmMAtUsI
+aMJnQuZ6R+KEpW/TR5qSKLWZSG45z/op+tzI2m+cE6HwTRVAWbcuJxcAA55MZjqU
+OOOu3BBYMjS5nf2sQ9uoXsVBFH7i0mQqoW1SLzr9opI8KsWwFxQmO2vBxWYaN+lH
+OmwBZBwyODIsmI1YGXmTp09NxRYz3Qe5GCgFzYowpMrcxUC24iduIdMwwhRM7rKg
+7GtIWMSrFfuI1XCLRmSlhDbhNN6fVg2f8Bo9PdH9ihiIyxSrc+FOUasUYCCJvlSZ
+8hFUlLvcmrZlWuazohm0lsXuMK1JflmQr/DA/uXxP9xzFfRy+RU3jDyxJbRHAgMB
+AAGjUzBRMB0GA1UdDgQWBBSQJyxiPMRK01i+0BsV9zUwDiBaHzAfBgNVHSMEGDAW
+gBSQJyxiPMRK01i+0BsV9zUwDiBaHzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3
+DQEBCwUAA4ICAQCR+7a7N/m+WLkxPPIA/CB4MOr2Uf8ixTv435Nyv6rXOun0+lTP
+ExSZ0uYQ+L0WylItI3cQHULldDueD+s8TGzxf5woaLKf6tqyr0NYhKs+UeNEzDnN
+9PHQIhX0SZw3XyXGUgPNBfRCg2ZDdtMMdOU4XlQN/IN/9hbYTrueyY7eXq9hmtI9
+1srftAMqr9SR1JP7aHI6DVgrEsZVMTDnfT8WmLSGLlY1HmGfdEn1Ip5sbo9uSkiH
+AEPgPfjYIvR5LqTOMn4KsrlZyBbFIDh9Sl99M1kZzgH6zUGVLCDg1y6Cms69fx/e
+W1HoIeVkY4b4TY7Bk7JsqyNhIuqu7ARaxkdaZWhYaA2YyknwANdFfNpfH+elCLIk
+BUt5S3f4i7DaUePTvKukCZiCq4Oyln7RcOn5If73wCeLB/ZM9Ei1HforyLWP1CN8
+XLfpHaoeoPSWIveI0XHUl65LsPN2UbMbul/F23hwl+h8+BLmyAS680Yhn4zEN6Ku
+B7Po90HoFa1Du3bmx4jsN73UkT/dwMTi6K072FbipnC1904oGlWmLwvAHvrtxxmL
+Pl3pvEaZIu8wa/PNF6Y7J7VIewikIJq6Ta6FrWeFfzMWOj2qA1ZZi6fUaDSNYvuV
+J5quYKCc/O+I/yDDf8wyBbZ/gvUXzUHTMYGG+bFrn1p7XDbYYeEJ6R/xEg==
+-----END CERTIFICATE-----
diff --git a/lib-python/2.7/test/selfsigned_pythontestdotnet.pem b/lib-python/2.7/test/selfsigned_pythontestdotnet.pem
--- a/lib-python/2.7/test/selfsigned_pythontestdotnet.pem
+++ b/lib-python/2.7/test/selfsigned_pythontestdotnet.pem
@@ -1,16 +1,34 @@
 -----BEGIN CERTIFICATE-----
-MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV
-BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u
-IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv
-bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG
-A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo
-b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0
-aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ
-Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm
-Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv
-EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl
-bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN
-AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h
-TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515
-C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM=
+MIIF9zCCA9+gAwIBAgIUH98b4Fw/DyugC9cV7VK7ZODzHsIwDQYJKoZIhvcNAQEL
+BQAwgYoxCzAJBgNVBAYTAlhZMRcwFQYDVQQIDA5DYXN0bGUgQW50aHJheDEYMBYG
+A1UEBwwPQXJndW1lbnQgQ2xpbmljMSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUg
+Rm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0aG9udGVzdC5uZXQw
+HhcNMTkwNTA4MDEwMjQzWhcNMjcwNzI0MDEwMjQzWjCBijELMAkGA1UEBhMCWFkx
+FzAVBgNVBAgMDkNhc3RsZSBBbnRocmF4MRgwFgYDVQQHDA9Bcmd1bWVudCBDbGlu
+aWMxIzAhBgNVBAoMGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMSMwIQYDVQQD
+DBpzZWxmLXNpZ25lZC5weXRob250ZXN0Lm5ldDCCAiIwDQYJKoZIhvcNAQEBBQAD
+ggIPADCCAgoCggIBAMKdJlyCThkahwoBb7pl5q64Pe9Fn5jrIvzsveHTc97TpjV2
+RLfICnXKrltPk/ohkVl6K5SUZQZwMVzFubkyxE0nZPHYHlpiKWQxbsYVkYv01rix
+IFdLvaxxbGYke2jwQao31s4o61AdlsfK1SdpHQUynBBMssqI3SB4XPmcA7e+wEEx
+jxjVish4ixA1vuIZOx8yibu+CFCf/geEjoBMF3QPdzULzlrCSw8k/45iZCSoNbvK
+DoL4TVV07PHOxpheDh8ZQmepGvU6pVqhb9m4lgmV0OGWHgozd5Ur9CbTVDmxIEz3
+TSoRtNJK7qtyZdGNqwjksQxgZTjM/d/Lm/BJG99AiOmYOjsl9gbQMZgvQmMAtUsI
+aMJnQuZ6R+KEpW/TR5qSKLWZSG45z/op+tzI2m+cE6HwTRVAWbcuJxcAA55MZjqU
+OOOu3BBYMjS5nf2sQ9uoXsVBFH7i0mQqoW1SLzr9opI8KsWwFxQmO2vBxWYaN+lH
+OmwBZBwyODIsmI1YGXmTp09NxRYz3Qe5GCgFzYowpMrcxUC24iduIdMwwhRM7rKg
+7GtIWMSrFfuI1XCLRmSlhDbhNN6fVg2f8Bo9PdH9ihiIyxSrc+FOUasUYCCJvlSZ
+8hFUlLvcmrZlWuazohm0lsXuMK1JflmQr/DA/uXxP9xzFfRy+RU3jDyxJbRHAgMB
+AAGjUzBRMB0GA1UdDgQWBBSQJyxiPMRK01i+0BsV9zUwDiBaHzAfBgNVHSMEGDAW
+gBSQJyxiPMRK01i+0BsV9zUwDiBaHzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3
+DQEBCwUAA4ICAQCR+7a7N/m+WLkxPPIA/CB4MOr2Uf8ixTv435Nyv6rXOun0+lTP
+ExSZ0uYQ+L0WylItI3cQHULldDueD+s8TGzxf5woaLKf6tqyr0NYhKs+UeNEzDnN
+9PHQIhX0SZw3XyXGUgPNBfRCg2ZDdtMMdOU4XlQN/IN/9hbYTrueyY7eXq9hmtI9
+1srftAMqr9SR1JP7aHI6DVgrEsZVMTDnfT8WmLSGLlY1HmGfdEn1Ip5sbo9uSkiH
+AEPgPfjYIvR5LqTOMn4KsrlZyBbFIDh9Sl99M1kZzgH6zUGVLCDg1y6Cms69fx/e
+W1HoIeVkY4b4TY7Bk7JsqyNhIuqu7ARaxkdaZWhYaA2YyknwANdFfNpfH+elCLIk
+BUt5S3f4i7DaUePTvKukCZiCq4Oyln7RcOn5If73wCeLB/ZM9Ei1HforyLWP1CN8
+XLfpHaoeoPSWIveI0XHUl65LsPN2UbMbul/F23hwl+h8+BLmyAS680Yhn4zEN6Ku
+B7Po90HoFa1Du3bmx4jsN73UkT/dwMTi6K072FbipnC1904oGlWmLwvAHvrtxxmL
+Pl3pvEaZIu8wa/PNF6Y7J7VIewikIJq6Ta6FrWeFfzMWOj2qA1ZZi6fUaDSNYvuV
+J5quYKCc/O+I/yDDf8wyBbZ/gvUXzUHTMYGG+bFrn1p7XDbYYeEJ6R/xEg==
 -----END CERTIFICATE-----
diff --git a/lib-python/2.7/test/test_ftplib.py b/lib-python/2.7/test/test_ftplib.py
--- a/lib-python/2.7/test/test_ftplib.py
+++ b/lib-python/2.7/test/test_ftplib.py
@@ -240,11 +240,17 @@
     def run(self):
         self.active = True
         self.__flag.set()
-        while self.active and asyncore.socket_map:
-            self.active_lock.acquire()
-            asyncore.loop(timeout=0.1, count=1)
-            self.active_lock.release()
-        asyncore.close_all(ignore_all=True)
+        try:
+            while self.active and asyncore.socket_map:
+                self.active_lock.acquire()
+                try:
+                    asyncore.loop(timeout=0.1, count=1)
+                except:
+                    self.active_lock.release()
+                    raise
+                self.active_lock.release()
+        finally:
+            asyncore.close_all(ignore_all=True)
 
     def stop(self):
         assert self.active
diff --git a/lib-python/2.7/test/test_sys.py b/lib-python/2.7/test/test_sys.py
--- a/lib-python/2.7/test/test_sys.py
+++ b/lib-python/2.7/test/test_sys.py
@@ -227,6 +227,11 @@
                 raise ValueError()
             except ValueError, e:
                 pass
+        except MemoryError:
+            # Documentation for setrecursionlimit says: "The highest possible
+            # limit is platform-dependent. ... a too-high limit can lead to a
+            # crash" so we allow MemoryError here
+            pass
         finally:
             sys.setrecursionlimit(oldlimit)
 
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -404,7 +404,7 @@
     RegrTest('test_source_encoding.py'),
     RegrTest('test_spwd.py'),
     RegrTest('test_sqlite.py', usemodules="thread _rawffi zlib"),
-    RegrTest('test_ssl.py', usemodules='_ssl _socket select'),
+    RegrTest('test_ssl.py', usemodules='_socket select'),
     RegrTest('test_startfile.py'),
     RegrTest('test_stat.py'),
     RegrTest('test_str.py', core=True),
diff --git a/lib_pypy/_cffi_ssl/LICENSE b/lib_pypy/_cffi_ssl/LICENSE
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/LICENSE
@@ -0,0 +1,26 @@
+
+Except when otherwise stated (look for LICENSE files in directories or
+information at the beginning of each file) all software and
+documentation is licensed as follows: 
+
+    The MIT License
+
+    Permission is hereby granted, free of charge, to any person 
+    obtaining a copy of this software and associated documentation 
+    files (the "Software"), to deal in the Software without 
+    restriction, including without limitation the rights to use, 
+    copy, modify, merge, publish, distribute, sublicense, and/or 
+    sell copies of the Software, and to permit persons to whom the 
+    Software is furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included 
+    in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+    OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
+    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+    DEALINGS IN THE SOFTWARE.
+
diff --git a/lib_pypy/_cffi_ssl/README.md b/lib_pypy/_cffi_ssl/README.md
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/README.md
@@ -0,0 +1,35 @@
+# PyPy's SSL module
+
+All of the CFFI code is copied from cryptography, wich patches contributed
+back to cryptography. PyPy vendors it's own copy of the cffi backend thus
+it renames the compiled shared object to _pypy_openssl.so (which means
+that cryptography can ship their own cffi backend)
+
+NOTE: currently, we have the following changes:
+
+* ``_cffi_src/openssl/callbacks.py`` to not rely on the CPython C API
+  (this change is now backported)
+
+* ``_cffi_src/utils.py`` for issue #2575 (29c9a89359e4)
+
+* ``_cffi_src/openssl/x509_vfy.py`` for issue #2605 (ca4d0c90f5a1)
+
+* ``_cffi_src/openssl/pypy_win32_extra.py`` for Win32-only functionality like ssl.enum_certificates()
+
+
+# Tests?
+
+Currently this module is tested using CPython's standard library test suite.
+
+# Install it into PyPy's source tree
+
+Copy over all the sources into the folder `lib_pypy/_cffi_ssl/*`. Updating the cffi backend can be simply done by the following command::
+
+    $ cp -r <cloned cryptography folder>/src/_cffi_src/* .
+
+NOTE: you need to keep our version of ``_cffi_src/openssl/callbacks.py``
+for now!
+
+# Crpytography version
+
+Copied over release version `1.7.2`
diff --git a/lib_pypy/_cffi_ssl/__init__.py b/lib_pypy/_cffi_ssl/__init__.py
new file mode 100644
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/__init__.py b/lib_pypy/_cffi_ssl/_cffi_src/__init__.py
new file mode 100644
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/build_commoncrypto.py b/lib_pypy/_cffi_ssl/_cffi_src/build_commoncrypto.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/build_commoncrypto.py
@@ -0,0 +1,33 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+from _cffi_src.utils import build_ffi_for_binding
+
+
+ffi = build_ffi_for_binding(
+    module_name="_commoncrypto",
+    module_prefix="_cffi_src.commoncrypto.",
+    modules=[
+        "cf",
+        "common_digest",
+        "common_hmac",
+        "common_key_derivation",
+        "common_cryptor",
+        "common_symmetric_key_wrap",
+        "seccertificate",
+        "secimport",
+        "secitem",
+        "seckey",
+        "seckeychain",
+        "secpolicy",
+        "sectransform",
+        "sectrust",
+        "secure_transport",
+    ],
+    extra_link_args=[
+        "-framework", "Security", "-framework", "CoreFoundation"
+    ],
+)
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/build_constant_time.py b/lib_pypy/_cffi_ssl/_cffi_src/build_constant_time.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/build_constant_time.py
@@ -0,0 +1,27 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+import os
+
+from _cffi_src.utils import build_ffi, compiler_type, extra_link_args
+
+
+with open(os.path.join(
+    os.path.dirname(__file__), "hazmat_src/constant_time.h"
+)) as f:
+    types = f.read()
+
+with open(os.path.join(
+    os.path.dirname(__file__), "hazmat_src/constant_time.c"
+)) as f:
+    functions = f.read()
+
+ffi = build_ffi(
+    module_name="_constant_time",
+    cdef_source=types,
+    verify_source=functions,
+    extra_link_args=extra_link_args(compiler_type()),
+)
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/build_openssl.py b/lib_pypy/_cffi_ssl/_cffi_src/build_openssl.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/build_openssl.py
@@ -0,0 +1,86 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+import os
+import sys
+
+from _cffi_src.utils import (
+    build_ffi_for_binding, compiler_type, extra_link_args
+)
+
+
+def _get_openssl_libraries(platform):
+    # OpenSSL goes by a different library name on different operating systems.
+    if platform == "darwin":
+        return _osx_libraries(
+            os.environ.get("CRYPTOGRAPHY_OSX_NO_LINK_FLAGS")
+        )
+    elif platform == "win32":
+        if compiler_type() == "msvc":
+            libs = ["libeay32", "ssleay32"]
+        else:
+            libs = ["ssl", "crypto"]
+        return libs + ["advapi32", "crypt32", "gdi32", "user32", "ws2_32"]
+    else:
+        # In some circumstances, the order in which these libs are
+        # specified on the linker command-line is significant;
+        # libssl must come before libcrypto
+        # (http://marc.info/?l=openssl-users&m=135361825921871)
+        return ["ssl", "crypto"]
+
+
+def _osx_libraries(build_static):
+    # For building statically we don't want to pass the -lssl or -lcrypto flags
+    if build_static == "1":
+        return []
+    else:
+        return ["ssl", "crypto"]
+
+
+ffi = build_ffi_for_binding(
+    module_name="_openssl",
+    module_prefix="_cffi_src.openssl.",
+    modules=[
+        # This goes first so we can define some cryptography-wide symbols.
+        "cryptography",
+
+        "aes",
+        "asn1",
+        "bignum",
+        "bio",
+        "cmac",
+        "cms",
+        "conf",
+        "crypto",
+        "dh",
+        "dsa",
+        "ec",
+        "ecdh",
+        "ecdsa",
+        "engine",
+        "err",
+        "evp",
+        "hmac",
+        "nid",
+        "objects",
+        "ocsp",
+        "opensslv",
+        "osrandom_engine",
+        "pem",
+        "pkcs12",
+        "rand",
+        "rsa",
+        "ssl",
+        "x509",
+        "x509name",
+        "x509v3",
+        "x509_vfy",
+        "pkcs7",
+        "callbacks",
+    ],
+    libraries=_get_openssl_libraries(sys.platform),
+    extra_link_args=extra_link_args(compiler_type()),
+)
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/build_padding.py b/lib_pypy/_cffi_ssl/_cffi_src/build_padding.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/build_padding.py
@@ -0,0 +1,27 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+import os
+
+from _cffi_src.utils import build_ffi, compiler_type, extra_link_args
+
+
+with open(os.path.join(
+    os.path.dirname(__file__), "hazmat_src/padding.h"
+)) as f:
+    types = f.read()
+
+with open(os.path.join(
+    os.path.dirname(__file__), "hazmat_src/padding.c"
+)) as f:
+    functions = f.read()
+
+ffi = build_ffi(
+    module_name="_padding",
+    cdef_source=types,
+    verify_source=functions,
+    extra_link_args=extra_link_args(compiler_type()),
+)
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/__init__.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/__init__.py
@@ -0,0 +1,5 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/cf.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/cf.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/cf.py
@@ -0,0 +1,113 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CoreFoundation/CoreFoundation.h>
+"""
+
+TYPES = """
+typedef bool Boolean;
+typedef signed long OSStatus;
+typedef unsigned char UInt8;
+typedef uint32_t UInt32;
+
+typedef const void * CFAllocatorRef;
+const CFAllocatorRef kCFAllocatorDefault;
+typedef ... *CFDataRef;
+typedef signed long long CFIndex;
+typedef ... *CFStringRef;
+typedef ... *CFArrayRef;
+typedef ... *CFMutableArrayRef;
+typedef ... *CFBooleanRef;
+typedef ... *CFErrorRef;
+typedef ... *CFNumberRef;
+typedef ... *CFTypeRef;
+typedef ... *CFDictionaryRef;
+typedef ... *CFMutableDictionaryRef;
+typedef struct {
+    ...;
+} CFDictionaryKeyCallBacks;
+typedef struct {
+    ...;
+} CFDictionaryValueCallBacks;
+typedef struct {
+    ...;
+} CFRange;
+typedef struct {
+    ...;
+} CFArrayCallBacks;
+
+typedef UInt32 CFStringEncoding;
+enum {
+    kCFStringEncodingASCII = 0x0600
+};
+
+enum {
+   kCFNumberSInt8Type = 1,
+   kCFNumberSInt16Type = 2,
+   kCFNumberSInt32Type = 3,
+   kCFNumberSInt64Type = 4,
+   kCFNumberFloat32Type = 5,
+   kCFNumberFloat64Type = 6,
+   kCFNumberCharType = 7,
+   kCFNumberShortType = 8,
+   kCFNumberIntType = 9,
+   kCFNumberLongType = 10,
+   kCFNumberLongLongType = 11,
+   kCFNumberFloatType = 12,
+   kCFNumberDoubleType = 13,
+   kCFNumberCFIndexType = 14,
+   kCFNumberNSIntegerType = 15,
+   kCFNumberCGFloatType = 16,
+   kCFNumberMaxType = 16
+};
+typedef int CFNumberType;
+
+const CFDictionaryKeyCallBacks kCFTypeDictionaryKeyCallBacks;
+const CFDictionaryValueCallBacks kCFTypeDictionaryValueCallBacks;
+
+const CFArrayCallBacks kCFTypeArrayCallBacks;
+
+const CFBooleanRef kCFBooleanTrue;
+const CFBooleanRef kCFBooleanFalse;
+"""
+
+FUNCTIONS = """
+CFDataRef CFDataCreate(CFAllocatorRef, const UInt8 *, CFIndex);
+CFStringRef CFStringCreateWithCString(CFAllocatorRef, const char *,
+                                      CFStringEncoding);
+CFDictionaryRef CFDictionaryCreate(CFAllocatorRef, const void **,
+                                   const void **, CFIndex,
+                                   const CFDictionaryKeyCallBacks *,
+                                   const CFDictionaryValueCallBacks *);
+CFMutableDictionaryRef CFDictionaryCreateMutable(
+    CFAllocatorRef,
+    CFIndex,
+    const CFDictionaryKeyCallBacks *,
+    const CFDictionaryValueCallBacks *
+);
+void CFDictionarySetValue(CFMutableDictionaryRef, const void *, const void *);
+CFIndex CFArrayGetCount(CFArrayRef);
+const void *CFArrayGetValueAtIndex(CFArrayRef, CFIndex);
+CFIndex CFDataGetLength(CFDataRef);
+void CFDataGetBytes(CFDataRef, CFRange, UInt8 *);
+CFRange CFRangeMake(CFIndex, CFIndex);
+void CFShow(CFTypeRef);
+Boolean CFBooleanGetValue(CFBooleanRef);
+CFNumberRef CFNumberCreate(CFAllocatorRef, CFNumberType, const void *);
+void CFRelease(CFTypeRef);
+CFTypeRef CFRetain(CFTypeRef);
+
+CFMutableArrayRef CFArrayCreateMutable(CFAllocatorRef, CFIndex,
+                                       const CFArrayCallBacks *);
+void CFArrayAppendValue(CFMutableArrayRef, const void *);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_cryptor.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_cryptor.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_cryptor.py
@@ -0,0 +1,99 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonCryptor.h>
+"""
+
+TYPES = """
+enum {
+    kCCAlgorithmAES128 = 0,
+    kCCAlgorithmDES,
+    kCCAlgorithm3DES,
+    kCCAlgorithmCAST,
+    kCCAlgorithmRC4,
+    kCCAlgorithmRC2,
+    kCCAlgorithmBlowfish
+};
+typedef uint32_t CCAlgorithm;
+enum {
+    kCCSuccess = 0,
+    kCCParamError = -4300,
+    kCCBufferTooSmall = -4301,
+    kCCMemoryFailure = -4302,
+    kCCAlignmentError = -4303,
+    kCCDecodeError = -4304,
+    kCCUnimplemented = -4305
+};
+typedef int32_t CCCryptorStatus;
+typedef uint32_t CCOptions;
+enum {
+    kCCEncrypt = 0,
+    kCCDecrypt,
+};
+typedef uint32_t CCOperation;
+typedef ... *CCCryptorRef;
+
+enum {
+    kCCModeOptionCTR_LE = 0x0001,
+    kCCModeOptionCTR_BE = 0x0002
+};
+
+typedef uint32_t CCModeOptions;
+
+enum {
+    kCCModeECB = 1,
+    kCCModeCBC = 2,
+    kCCModeCFB = 3,
+    kCCModeCTR = 4,
+    kCCModeF8 = 5,
+    kCCModeLRW = 6,
+    kCCModeOFB = 7,
+    kCCModeXTS = 8,
+    kCCModeRC4 = 9,
+    kCCModeCFB8 = 10,
+    kCCModeGCM = 11
+};
+typedef uint32_t CCMode;
+enum {
+    ccNoPadding = 0,
+    ccPKCS7Padding = 1,
+};
+typedef uint32_t CCPadding;
+"""
+
+FUNCTIONS = """
+CCCryptorStatus CCCryptorCreateWithMode(CCOperation, CCMode, CCAlgorithm,
+                                        CCPadding, const void *, const void *,
+                                        size_t, const void *, size_t, int,
+                                        CCModeOptions, CCCryptorRef *);
+CCCryptorStatus CCCryptorCreate(CCOperation, CCAlgorithm, CCOptions,
+                                const void *, size_t, const void *,
+                                CCCryptorRef *);
+CCCryptorStatus CCCryptorUpdate(CCCryptorRef, const void *, size_t, void *,
+                                size_t, size_t *);
+CCCryptorStatus CCCryptorFinal(CCCryptorRef, void *, size_t, size_t *);
+CCCryptorStatus CCCryptorRelease(CCCryptorRef);
+
+CCCryptorStatus CCCryptorGCMAddIV(CCCryptorRef, const void *, size_t);
+CCCryptorStatus CCCryptorGCMAddAAD(CCCryptorRef, const void *, size_t);
+CCCryptorStatus CCCryptorGCMEncrypt(CCCryptorRef, const void *, size_t,
+                                    void *);
+CCCryptorStatus CCCryptorGCMDecrypt(CCCryptorRef, const void *, size_t,
+                                    void *);
+CCCryptorStatus CCCryptorGCMFinal(CCCryptorRef, const void *, size_t *);
+CCCryptorStatus CCCryptorGCMReset(CCCryptorRef);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+/* Not defined in the public header */
+enum {
+    kCCModeGCM = 11
+};
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_digest.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_digest.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_digest.py
@@ -0,0 +1,58 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonDigest.h>
+"""
+
+TYPES = """
+typedef uint32_t CC_LONG;
+typedef uint64_t CC_LONG64;
+typedef struct CC_MD5state_st {
+    ...;
+} CC_MD5_CTX;
+typedef struct CC_SHA1state_st {
+    ...;
+} CC_SHA1_CTX;
+typedef struct CC_SHA256state_st {
+    ...;
+} CC_SHA256_CTX;
+typedef struct CC_SHA512state_st {
+    ...;
+} CC_SHA512_CTX;
+"""
+
+FUNCTIONS = """
+int CC_MD5_Init(CC_MD5_CTX *);
+int CC_MD5_Update(CC_MD5_CTX *, const void *, CC_LONG);
+int CC_MD5_Final(unsigned char *, CC_MD5_CTX *);
+
+int CC_SHA1_Init(CC_SHA1_CTX *);
+int CC_SHA1_Update(CC_SHA1_CTX *, const void *, CC_LONG);
+int CC_SHA1_Final(unsigned char *, CC_SHA1_CTX *);
+
+int CC_SHA224_Init(CC_SHA256_CTX *);
+int CC_SHA224_Update(CC_SHA256_CTX *, const void *, CC_LONG);
+int CC_SHA224_Final(unsigned char *, CC_SHA256_CTX *);
+
+int CC_SHA256_Init(CC_SHA256_CTX *);
+int CC_SHA256_Update(CC_SHA256_CTX *, const void *, CC_LONG);
+int CC_SHA256_Final(unsigned char *, CC_SHA256_CTX *);
+
+int CC_SHA384_Init(CC_SHA512_CTX *);
+int CC_SHA384_Update(CC_SHA512_CTX *, const void *, CC_LONG);
+int CC_SHA384_Final(unsigned char *, CC_SHA512_CTX *);
+
+int CC_SHA512_Init(CC_SHA512_CTX *);
+int CC_SHA512_Update(CC_SHA512_CTX *, const void *, CC_LONG);
+int CC_SHA512_Final(unsigned char *, CC_SHA512_CTX *);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_hmac.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_hmac.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_hmac.py
@@ -0,0 +1,37 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonHMAC.h>
+"""
+
+TYPES = """
+typedef struct {
+    ...;
+} CCHmacContext;
+enum {
+    kCCHmacAlgSHA1,
+    kCCHmacAlgMD5,
+    kCCHmacAlgSHA256,
+    kCCHmacAlgSHA384,
+    kCCHmacAlgSHA512,
+    kCCHmacAlgSHA224
+};
+typedef uint32_t CCHmacAlgorithm;
+"""
+
+FUNCTIONS = """
+void CCHmacInit(CCHmacContext *, CCHmacAlgorithm, const void *, size_t);
+void CCHmacUpdate(CCHmacContext *, const void *, size_t);
+void CCHmacFinal(CCHmacContext *, void *);
+
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_key_derivation.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_key_derivation.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_key_derivation.py
@@ -0,0 +1,39 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonKeyDerivation.h>
+"""
+
+TYPES = """
+enum {
+    kCCPBKDF2 = 2,
+};
+typedef uint32_t CCPBKDFAlgorithm;
+enum {
+    kCCPRFHmacAlgSHA1 = 1,
+    kCCPRFHmacAlgSHA224 = 2,
+    kCCPRFHmacAlgSHA256 = 3,
+    kCCPRFHmacAlgSHA384 = 4,
+    kCCPRFHmacAlgSHA512 = 5,
+};
+typedef uint32_t CCPseudoRandomAlgorithm;
+typedef unsigned int uint;
+"""
+
+FUNCTIONS = """
+int CCKeyDerivationPBKDF(CCPBKDFAlgorithm, const char *, size_t,
+                         const uint8_t *, size_t, CCPseudoRandomAlgorithm,
+                         uint, uint8_t *, size_t);
+uint CCCalibratePBKDF(CCPBKDFAlgorithm, size_t, size_t,
+                      CCPseudoRandomAlgorithm, size_t, uint32_t);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_symmetric_key_wrap.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_symmetric_key_wrap.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_symmetric_key_wrap.py
@@ -0,0 +1,35 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonSymmetricKeywrap.h>
+"""
+
+TYPES = """
+enum {
+    kCCWRAPAES = 1,
+};
+
+typedef uint32_t CCWrappingAlgorithm;
+"""
+
+FUNCTIONS = """
+int CCSymmetricKeyWrap(CCWrappingAlgorithm, const uint8_t *, const size_t,
+                        const uint8_t *, size_t, const uint8_t *, size_t,
+                        uint8_t *, size_t *);
+int CCSymmetricKeyUnwrap(CCWrappingAlgorithm algorithm, const uint8_t *,
+                         const size_t, const uint8_t *, size_t,
+                         const uint8_t *, size_t, uint8_t *, size_t *);
+size_t CCSymmetricWrappedSize(CCWrappingAlgorithm, size_t);
+size_t CCSymmetricUnwrappedSize(CCWrappingAlgorithm, size_t);
+
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seccertificate.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seccertificate.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seccertificate.py
@@ -0,0 +1,23 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecCertificate.h>
+"""
+
+TYPES = """
+typedef ... *SecCertificateRef;
+"""
+
+FUNCTIONS = """
+SecCertificateRef SecCertificateCreateWithData(CFAllocatorRef, CFDataRef);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secimport.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secimport.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secimport.py
@@ -0,0 +1,86 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecImportExport.h>
+"""
+
+TYPES = """
+typedef ... *SecAccessRef;
+
+CFStringRef kSecImportExportPassphrase;
+CFStringRef kSecImportExportKeychain;
+CFStringRef kSecImportExportAccess;
+
+typedef uint32_t SecExternalItemType;
+enum {
+    kSecItemTypeUnknown,
+    kSecItemTypePrivateKey,
+    kSecItemTypePublicKey,
+    kSecItemTypeSessionKey,
+    kSecItemTypeCertificate,
+    kSecItemTypeAggregate
+};
+
+
+typedef uint32_t SecExternalFormat;
+enum {
+    kSecFormatUnknown = 0,
+    kSecFormatOpenSSL,
+    kSecFormatSSH,
+    kSecFormatBSAFE,
+    kSecFormatRawKey,
+    kSecFormatWrappedPKCS8,
+    kSecFormatWrappedOpenSSL,
+    kSecFormatWrappedSSH,
+    kSecFormatWrappedLSH,
+    kSecFormatX509Cert,
+    kSecFormatPEMSequence,
+    kSecFormatPKCS7,
+    kSecFormatPKCS12,
+    kSecFormatNetscapeCertSequence,
+    kSecFormatSSHv2
+};
+
+typedef uint32_t SecItemImportExportFlags;
+enum {
+    kSecKeyImportOnlyOne        = 0x00000001,
+    kSecKeySecurePassphrase     = 0x00000002,
+    kSecKeyNoAccessControl      = 0x00000004
+};
+typedef uint32_t SecKeyImportExportFlags;
+
+typedef struct {
+    /* for import and export */
+    uint32_t version;
+    SecKeyImportExportFlags  flags;
+    CFTypeRef                passphrase;
+    CFStringRef              alertTitle;
+    CFStringRef              alertPrompt;
+
+    /* for import only */
+    SecAccessRef             accessRef;
+    CFArrayRef               keyUsage;
+
+    CFArrayRef               keyAttributes;
+} SecItemImportExportKeyParameters;
+"""
+
+FUNCTIONS = """
+OSStatus SecItemImport(CFDataRef, CFStringRef, SecExternalFormat *,
+                       SecExternalItemType *, SecItemImportExportFlags,
+                       const SecItemImportExportKeyParameters *,
+                       SecKeychainRef, CFArrayRef *);
+OSStatus SecPKCS12Import(CFDataRef, CFDictionaryRef, CFArrayRef *);
+OSStatus SecItemExport(CFTypeRef, SecExternalFormat, SecItemImportExportFlags,
+                       const SecItemImportExportKeyParameters *, CFDataRef *);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secitem.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secitem.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secitem.py
@@ -0,0 +1,27 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecItem.h>
+"""
+
+TYPES = """
+const CFTypeRef kSecAttrKeyType;
+const CFTypeRef kSecAttrKeySizeInBits;
+const CFTypeRef kSecAttrIsPermanent;
+const CFTypeRef kSecAttrKeyTypeRSA;
+const CFTypeRef kSecAttrKeyTypeDSA;
+const CFTypeRef kSecUseKeychain;
+"""
+
+FUNCTIONS = """
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seckey.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seckey.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seckey.py
@@ -0,0 +1,24 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecKey.h>
+"""
+
+TYPES = """
+typedef ... *SecKeyRef;
+"""
+
+FUNCTIONS = """
+OSStatus SecKeyGeneratePair(CFDictionaryRef, SecKeyRef *, SecKeyRef *);
+size_t SecKeyGetBlockSize(SecKeyRef);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seckeychain.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seckeychain.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seckeychain.py
@@ -0,0 +1,25 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecKeychain.h>
+"""
+
+TYPES = """
+typedef ... *SecKeychainRef;
+"""
+
+FUNCTIONS = """
+OSStatus SecKeychainCreate(const char *, UInt32, const void *, Boolean,
+                           SecAccessRef, SecKeychainRef *);
+OSStatus SecKeychainDelete(SecKeychainRef);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secpolicy.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secpolicy.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secpolicy.py
@@ -0,0 +1,23 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecPolicy.h>
+"""
+
+TYPES = """
+typedef ... *SecPolicyRef;
+"""
+
+FUNCTIONS = """
+SecPolicyRef SecPolicyCreateSSL(Boolean, CFStringRef);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/sectransform.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/sectransform.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/sectransform.py
@@ -0,0 +1,68 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecDigestTransform.h>
+#include <Security/SecSignVerifyTransform.h>
+#include <Security/SecEncryptTransform.h>
+"""
+
+TYPES = """
+typedef ... *SecTransformRef;
+
+CFStringRef kSecImportExportPassphrase;
+CFStringRef kSecImportExportKeychain;
+CFStringRef kSecImportExportAccess;
+
+CFStringRef kSecEncryptionMode;
+CFStringRef kSecEncryptKey;
+CFStringRef kSecIVKey;
+CFStringRef kSecModeCBCKey;
+CFStringRef kSecModeCFBKey;
+CFStringRef kSecModeECBKey;
+CFStringRef kSecModeNoneKey;
+CFStringRef kSecModeOFBKey;
+CFStringRef kSecOAEPEncodingParametersAttributeName;
+CFStringRef kSecPaddingKey;
+CFStringRef kSecPaddingNoneKey;
+CFStringRef kSecPaddingOAEPKey;
+CFStringRef kSecPaddingPKCS1Key;
+CFStringRef kSecPaddingPKCS5Key;
+CFStringRef kSecPaddingPKCS7Key;
+
+const CFStringRef kSecTransformInputAttributeName;
+const CFStringRef kSecTransformOutputAttributeName;
+const CFStringRef kSecTransformDebugAttributeName;
+const CFStringRef kSecTransformTransformName;
+const CFStringRef kSecTransformAbortAttributeName;
+
+CFStringRef kSecInputIsAttributeName;
+CFStringRef kSecInputIsPlainText;
+CFStringRef kSecInputIsDigest;
+CFStringRef kSecInputIsRaw;
+
+const CFStringRef kSecDigestTypeAttribute;
+const CFStringRef kSecDigestLengthAttribute;
+const CFStringRef kSecDigestMD5;
+const CFStringRef kSecDigestSHA1;
+const CFStringRef kSecDigestSHA2;
+"""
+
+FUNCTIONS = """
+Boolean SecTransformSetAttribute(SecTransformRef, CFStringRef, CFTypeRef,
+                                 CFErrorRef *);
+SecTransformRef SecDecryptTransformCreate(SecKeyRef, CFErrorRef *);
+SecTransformRef SecEncryptTransformCreate(SecKeyRef, CFErrorRef *);
+SecTransformRef SecVerifyTransformCreate(SecKeyRef, CFDataRef, CFErrorRef *);
+SecTransformRef SecSignTransformCreate(SecKeyRef, CFErrorRef *) ;
+CFTypeRef SecTransformExecute(SecTransformRef, CFErrorRef *);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/sectrust.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/sectrust.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/sectrust.py
@@ -0,0 +1,39 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecTrust.h>
+"""
+
+TYPES = """
+typedef ... *SecTrustRef;
+typedef uint32_t SecTrustResultType;
+
+enum {
+    kSecTrustResultInvalid,
+    kSecTrustResultProceed,
+    kSecTrustResultDeny,
+    kSecTrustResultUnspecified,
+    kSecTrustResultRecoverableTrustFailure,
+    kSecTrustResultFatalTrustFailure,
+    kSecTrustResultOtherError
+};
+"""
+
+FUNCTIONS = """
+OSStatus SecTrustEvaluate(SecTrustRef, SecTrustResultType *);
+OSStatus SecTrustCopyAnchorCertificates(CFArrayRef *);
+"""
+
+MACROS = """
+/* The first argument changed from CFArrayRef to CFTypeRef in 10.8, so this
+ * has to go here for compatibility.
+ */
+OSStatus SecTrustCreateWithCertificates(CFTypeRef, CFTypeRef, SecTrustRef *);
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secure_transport.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secure_transport.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secure_transport.py
@@ -0,0 +1,308 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecureTransport.h>
+"""
+
+TYPES = """
+typedef ... *SSLContextRef;
+typedef const void *SSLConnectionRef;
+
+typedef enum {
+    kSSLSessionOptionBreakOnServerAuth,
+    kSSLSessionOptionBreakOnCertRequested,
+} SSLSessionOption;
+
+typedef enum {
+    kNeverAuthenticate,
+    kAlwaysAuthenticate,
+    kTryAuthenticate
+} SSLAuthenticate;
+
+typedef enum {
+    kSSLIdle,
+    kSSLHandshake,
+    kSSLConnected,
+    kSSLClosed,
+    kSSLAborted
+} SSLSessionState;
+
+typedef enum {
+    kSSLProtocolUnknown = 0,
+    kSSLProtocol3       = 2,
+    kTLSProtocol1       = 4,
+    /* DEPRECATED on iOS */
+    kSSLProtocol2       = 1,
+    kSSLProtocol3Only   = 3,
+    kTLSProtocol1Only   = 5,
+    kSSLProtocolAll     = 6,
+} SSLProtocol;
+
+typedef UInt32 SSLCipherSuite;
+enum {
+   SSL_NULL_WITH_NULL_NULL =               0x0000,
+   SSL_RSA_WITH_NULL_MD5 =                 0x0001,
+   SSL_RSA_WITH_NULL_SHA =                 0x0002,
+   SSL_RSA_EXPORT_WITH_RC4_40_MD5 =        0x0003,
+   SSL_RSA_WITH_RC4_128_MD5 =              0x0004,
+   SSL_RSA_WITH_RC4_128_SHA =              0x0005,
+   SSL_RSA_EXPORT_WITH_RC2_CBC_40_MD5 =    0x0006,
+   SSL_RSA_WITH_IDEA_CBC_SHA =             0x0007,
+   SSL_RSA_EXPORT_WITH_DES40_CBC_SHA =     0x0008,
+   SSL_RSA_WITH_DES_CBC_SHA =              0x0009,
+   SSL_RSA_WITH_3DES_EDE_CBC_SHA =         0x000A,
+   SSL_DH_DSS_EXPORT_WITH_DES40_CBC_SHA =  0x000B,
+   SSL_DH_DSS_WITH_DES_CBC_SHA =           0x000C,
+   SSL_DH_DSS_WITH_3DES_EDE_CBC_SHA =      0x000D,
+   SSL_DH_RSA_EXPORT_WITH_DES40_CBC_SHA =  0x000E,
+   SSL_DH_RSA_WITH_DES_CBC_SHA =           0x000F,
+   SSL_DH_RSA_WITH_3DES_EDE_CBC_SHA =      0x0010,
+   SSL_DHE_DSS_EXPORT_WITH_DES40_CBC_SHA = 0x0011,
+   SSL_DHE_DSS_WITH_DES_CBC_SHA =          0x0012,
+   SSL_DHE_DSS_WITH_3DES_EDE_CBC_SHA =     0x0013,
+   SSL_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA = 0x0014,
+   SSL_DHE_RSA_WITH_DES_CBC_SHA =          0x0015,
+   SSL_DHE_RSA_WITH_3DES_EDE_CBC_SHA =     0x0016,
+   SSL_DH_anon_EXPORT_WITH_RC4_40_MD5 =    0x0017,
+   SSL_DH_anon_WITH_RC4_128_MD5 =          0x0018,
+   SSL_DH_anon_EXPORT_WITH_DES40_CBC_SHA = 0x0019,
+   SSL_DH_anon_WITH_DES_CBC_SHA =          0x001A,
+   SSL_DH_anon_WITH_3DES_EDE_CBC_SHA =     0x001B,
+   SSL_FORTEZZA_DMS_WITH_NULL_SHA =        0x001C,
+   SSL_FORTEZZA_DMS_WITH_FORTEZZA_CBC_SHA =0x001D,
+
+   /* TLS addenda using AES, per RFC 3268 */
+   TLS_RSA_WITH_AES_128_CBC_SHA      =     0x002F,
+   TLS_DH_DSS_WITH_AES_128_CBC_SHA   =     0x0030,
+   TLS_DH_RSA_WITH_AES_128_CBC_SHA   =     0x0031,
+   TLS_DHE_DSS_WITH_AES_128_CBC_SHA  =     0x0032,
+   TLS_DHE_RSA_WITH_AES_128_CBC_SHA  =     0x0033,
+   TLS_DH_anon_WITH_AES_128_CBC_SHA  =     0x0034,
+   TLS_RSA_WITH_AES_256_CBC_SHA      =     0x0035,
+   TLS_DH_DSS_WITH_AES_256_CBC_SHA   =     0x0036,
+   TLS_DH_RSA_WITH_AES_256_CBC_SHA   =     0x0037,
+   TLS_DHE_DSS_WITH_AES_256_CBC_SHA  =     0x0038,
+   TLS_DHE_RSA_WITH_AES_256_CBC_SHA  =     0x0039,
+   TLS_DH_anon_WITH_AES_256_CBC_SHA  =     0x003A,
+
+   /* ECDSA addenda, RFC 4492 */
+   TLS_ECDH_ECDSA_WITH_NULL_SHA           =    0xC001,
+   TLS_ECDH_ECDSA_WITH_RC4_128_SHA        =    0xC002,
+   TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA   =    0xC003,
+   TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA    =    0xC004,
+   TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA    =    0xC005,
+   TLS_ECDHE_ECDSA_WITH_NULL_SHA          =    0xC006,
+   TLS_ECDHE_ECDSA_WITH_RC4_128_SHA       =    0xC007,
+   TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA  =    0xC008,
+   TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA   =    0xC009,
+   TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA   =    0xC00A,
+   TLS_ECDH_RSA_WITH_NULL_SHA             =    0xC00B,
+   TLS_ECDH_RSA_WITH_RC4_128_SHA          =    0xC00C,
+   TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA     =    0xC00D,
+   TLS_ECDH_RSA_WITH_AES_128_CBC_SHA      =    0xC00E,
+   TLS_ECDH_RSA_WITH_AES_256_CBC_SHA      =    0xC00F,
+   TLS_ECDHE_RSA_WITH_NULL_SHA            =    0xC010,
+   TLS_ECDHE_RSA_WITH_RC4_128_SHA         =    0xC011,
+   TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA    =    0xC012,
+   TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA     =    0xC013,
+   TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA     =    0xC014,
+   TLS_ECDH_anon_WITH_NULL_SHA            =    0xC015,
+   TLS_ECDH_anon_WITH_RC4_128_SHA         =    0xC016,
+   TLS_ECDH_anon_WITH_3DES_EDE_CBC_SHA    =    0xC017,
+   TLS_ECDH_anon_WITH_AES_128_CBC_SHA     =    0xC018,
+   TLS_ECDH_anon_WITH_AES_256_CBC_SHA     =    0xC019,
+
+   /* TLS 1.2 addenda, RFC 5246 */
+   /* Initial state. */
+   TLS_NULL_WITH_NULL_NULL                   = 0x0000,
+
+   /* Server provided RSA certificate for key exchange. */
+   TLS_RSA_WITH_NULL_MD5                     = 0x0001,
+   TLS_RSA_WITH_NULL_SHA                     = 0x0002,
+   TLS_RSA_WITH_RC4_128_MD5                  = 0x0004,
+   TLS_RSA_WITH_RC4_128_SHA                  = 0x0005,
+   TLS_RSA_WITH_3DES_EDE_CBC_SHA             = 0x000A,
+   TLS_RSA_WITH_NULL_SHA256                  = 0x003B,
+   TLS_RSA_WITH_AES_128_CBC_SHA256           = 0x003C,
+   TLS_RSA_WITH_AES_256_CBC_SHA256           = 0x003D,
+
+   /* Server-authenticated (and optionally client-authenticated)
+      Diffie-Hellman. */
+   TLS_DH_DSS_WITH_3DES_EDE_CBC_SHA          = 0x000D,
+   TLS_DH_RSA_WITH_3DES_EDE_CBC_SHA          = 0x0010,
+   TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA         = 0x0013,
+   TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA         = 0x0016,
+   TLS_DH_DSS_WITH_AES_128_CBC_SHA256        = 0x003E,
+   TLS_DH_RSA_WITH_AES_128_CBC_SHA256        = 0x003F,
+   TLS_DHE_DSS_WITH_AES_128_CBC_SHA256       = 0x0040,
+   TLS_DHE_RSA_WITH_AES_128_CBC_SHA256       = 0x0067,
+   TLS_DH_DSS_WITH_AES_256_CBC_SHA256        = 0x0068,
+   TLS_DH_RSA_WITH_AES_256_CBC_SHA256        = 0x0069,
+   TLS_DHE_DSS_WITH_AES_256_CBC_SHA256       = 0x006A,
+   TLS_DHE_RSA_WITH_AES_256_CBC_SHA256       = 0x006B,
+
+   /* Completely anonymous Diffie-Hellman */
+   TLS_DH_anon_WITH_RC4_128_MD5              = 0x0018,
+   TLS_DH_anon_WITH_3DES_EDE_CBC_SHA         = 0x001B,
+   TLS_DH_anon_WITH_AES_128_CBC_SHA256       = 0x006C,
+   TLS_DH_anon_WITH_AES_256_CBC_SHA256       = 0x006D,
+
+   /* Addenda from rfc 5288 AES Galois Counter Mode (GCM) Cipher Suites
+      for TLS. */
+   TLS_RSA_WITH_AES_128_GCM_SHA256           = 0x009C,
+   TLS_RSA_WITH_AES_256_GCM_SHA384           = 0x009D,
+   TLS_DHE_RSA_WITH_AES_128_GCM_SHA256       = 0x009E,
+   TLS_DHE_RSA_WITH_AES_256_GCM_SHA384       = 0x009F,
+   TLS_DH_RSA_WITH_AES_128_GCM_SHA256        = 0x00A0,
+   TLS_DH_RSA_WITH_AES_256_GCM_SHA384        = 0x00A1,
+   TLS_DHE_DSS_WITH_AES_128_GCM_SHA256       = 0x00A2,
+   TLS_DHE_DSS_WITH_AES_256_GCM_SHA384       = 0x00A3,
+   TLS_DH_DSS_WITH_AES_128_GCM_SHA256        = 0x00A4,
+   TLS_DH_DSS_WITH_AES_256_GCM_SHA384        = 0x00A5,
+   TLS_DH_anon_WITH_AES_128_GCM_SHA256       = 0x00A6,
+   TLS_DH_anon_WITH_AES_256_GCM_SHA384       = 0x00A7,
+
+   /* Addenda from rfc 5289  Elliptic Curve Cipher Suites with
+      HMAC SHA-256/384. */
+   TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256   = 0xC023,
+   TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384   = 0xC024,
+   TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256    = 0xC025,
+   TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384    = 0xC026,
+   TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256     = 0xC027,
+   TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384     = 0xC028,
+   TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256      = 0xC029,
+   TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384      = 0xC02A,
+
+   /* Addenda from rfc 5289  Elliptic Curve Cipher Suites with
+      SHA-256/384 and AES Galois Counter Mode (GCM) */
+   TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256   = 0xC02B,
+   TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384   = 0xC02C,
+   TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256    = 0xC02D,
+   TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384    = 0xC02E,
+   TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256     = 0xC02F,
+   TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384     = 0xC030,
+   TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256      = 0xC031,
+   TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384      = 0xC032,
+
+   /* RFC 5746 - Secure Renegotiation */
+   TLS_EMPTY_RENEGOTIATION_INFO_SCSV         = 0x00FF,
+
+   /*
+   * Tags for SSL 2 cipher kinds that are not specified
+   * for SSL 3.
+   */
+   SSL_RSA_WITH_RC2_CBC_MD5 =              0xFF80,
+   SSL_RSA_WITH_IDEA_CBC_MD5 =             0xFF81,
+   SSL_RSA_WITH_DES_CBC_MD5 =              0xFF82,
+   SSL_RSA_WITH_3DES_EDE_CBC_MD5 =         0xFF83,
+   SSL_NO_SUCH_CIPHERSUITE =               0xFFFF
+};
+
+typedef enum {
+    kSSLClientCertNone,
+    kSSLClientCertRequested,
+    kSSLClientCertSent,
+    kSSLClientCertRejected
+} SSLClientCertificateState;
+
+enum {
+    errSSLProtocol              = -9800,
+    errSSLNegotiation           = -9801,
+    errSSLFatalAlert            = -9802,
+    errSSLWouldBlock            = -9803,
+    errSSLSessionNotFound       = -9804,
+    errSSLClosedGraceful        = -9805,
+    errSSLClosedAbort           = -9806,
+    errSSLXCertChainInvalid     = -9807,
+    errSSLBadCert               = -9808,
+    errSSLCrypto                = -9809,
+    errSSLInternal              = -9810,
+    errSSLModuleAttach          = -9811,
+    errSSLUnknownRootCert       = -9812,
+    errSSLNoRootCert            = -9813,
+    errSSLCertExpired           = -9814,
+    errSSLCertNotYetValid       = -9815,
+    errSSLClosedNoNotify        = -9816,
+    errSSLBufferOverflow        = -9817,
+    errSSLBadCipherSuite        = -9818,
+    errSSLPeerUnexpectedMsg     = -9819,
+    errSSLPeerBadRecordMac      = -9820,
+    errSSLPeerDecryptionFail    = -9821,
+    errSSLPeerRecordOverflow    = -9822,
+    errSSLPeerDecompressFail    = -9823,
+    errSSLPeerHandshakeFail     = -9824,
+    errSSLPeerBadCert           = -9825,
+    errSSLPeerUnsupportedCert   = -9826,
+    errSSLPeerCertRevoked       = -9827,
+    errSSLPeerCertExpired       = -9828,
+    errSSLPeerCertUnknown       = -9829,
+    errSSLIllegalParam          = -9830,
+    errSSLPeerUnknownCA         = -9831,
+    errSSLPeerAccessDenied      = -9832,
+    errSSLPeerDecodeError       = -9833,
+    errSSLPeerDecryptError      = -9834,
+    errSSLPeerExportRestriction = -9835,
+    errSSLPeerProtocolVersion   = -9836,
+    errSSLPeerInsufficientSecurity = -9837,
+    errSSLPeerInternalError     = -9838,
+    errSSLPeerUserCancelled     = -9839,
+    errSSLPeerNoRenegotiation   = -9840,
+    errSSLServerAuthCompleted   = -9841,
+    errSSLClientCertRequested   = -9842,
+    errSSLHostNameMismatch      = -9843,
+    errSSLConnectionRefused     = -9844,
+    errSSLDecryptionFail        = -9845,
+    errSSLBadRecordMac          = -9846,
+    errSSLRecordOverflow        = -9847,
+    errSSLBadConfiguration      = -9848,
+    errSSLLast                  = -9849     /* end of range, to be deleted */
+};
+"""
+
+FUNCTIONS = """
+OSStatus SSLSetConnection(SSLContextRef, SSLConnectionRef);
+OSStatus SSLGetConnection(SSLContextRef, SSLConnectionRef *);
+OSStatus SSLSetSessionOption(SSLContextRef, SSLSessionOption, Boolean);
+OSStatus SSLSetClientSideAuthenticate(SSLContextRef, SSLAuthenticate);
+
+OSStatus SSLHandshake(SSLContextRef);
+OSStatus SSLGetSessionState(SSLContextRef, SSLSessionState *);
+OSStatus SSLGetNegotiatedProtocolVersion(SSLContextRef, SSLProtocol *);
+OSStatus SSLSetPeerID(SSLContextRef, const void *, size_t);
+OSStatus SSLGetPeerID(SSLContextRef, const void **, size_t *);
+OSStatus SSLGetBufferedReadSize(SSLContextRef, size_t *);
+OSStatus SSLRead(SSLContextRef, void *, size_t, size_t *);
+OSStatus SSLWrite(SSLContextRef, const void *, size_t, size_t *);
+OSStatus SSLClose(SSLContextRef);
+
+OSStatus SSLGetNumberSupportedCiphers(SSLContextRef, size_t *);
+OSStatus SSLGetSupportedCiphers(SSLContextRef, SSLCipherSuite *, size_t *);
+OSStatus SSLSetEnabledCiphers(SSLContextRef, const SSLCipherSuite *, size_t);
+OSStatus SSLGetNumberEnabledCiphers(SSLContextRef, size_t *);
+OSStatus SSLGetEnabledCiphers(SSLContextRef, SSLCipherSuite *, size_t *);
+OSStatus SSLGetNegotiatedCipher(SSLContextRef, SSLCipherSuite *);
+OSStatus SSLSetDiffieHellmanParams(SSLContextRef, const void *, size_t);
+OSStatus SSLGetDiffieHellmanParams(SSLContextRef, const void **, size_t *);
+
+OSStatus SSLSetCertificateAuthorities(SSLContextRef, CFTypeRef, Boolean);
+OSStatus SSLCopyCertificateAuthorities(SSLContextRef, CFArrayRef *);
+OSStatus SSLCopyDistinguishedNames(SSLContextRef, CFArrayRef *);
+OSStatus SSLSetCertificate(SSLContextRef, CFArrayRef);
+OSStatus SSLGetClientCertificateState(SSLContextRef,
+                                      SSLClientCertificateState *);
+OSStatus SSLCopyPeerTrust(SSLContextRef, SecTrustRef *trust);
+
+OSStatus SSLSetPeerDomainName(SSLContextRef, const char *, size_t);
+OSStatus SSLGetPeerDomainNameLength(SSLContextRef, size_t *);
+OSStatus SSLGetPeerDomainName(SSLContextRef, char *, size_t *);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/constant_time.c b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/constant_time.c
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/constant_time.c
@@ -0,0 +1,22 @@
+// This file is dual licensed under the terms of the Apache License, Version
+// 2.0, and the BSD License. See the LICENSE file in the root of this
+// repository for complete details.
+
+uint8_t Cryptography_constant_time_bytes_eq(uint8_t *a, size_t len_a,
+                                            uint8_t *b, size_t len_b) {
+    size_t i = 0;
+    uint8_t mismatch = 0;
+    if (len_a != len_b) {
+        return 0;
+    }
+    for (i = 0; i < len_a; i++) {
+        mismatch |= a[i] ^ b[i];
+    }
+
+    /* Make sure any bits set are copied to the lowest bit */
+    mismatch |= mismatch >> 4;
+    mismatch |= mismatch >> 2;
+    mismatch |= mismatch >> 1;
+    /* Now check the low bit to see if it's set */
+    return (mismatch & 1) == 0;
+}
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/constant_time.h b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/constant_time.h
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/constant_time.h
@@ -0,0 +1,6 @@
+// This file is dual licensed under the terms of the Apache License, Version
+// 2.0, and the BSD License. See the LICENSE file in the root of this
+// repository for complete details.
+
+uint8_t Cryptography_constant_time_bytes_eq(uint8_t *, size_t, uint8_t *,
+                                            size_t);
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/padding.c b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/padding.c
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/padding.c
@@ -0,0 +1,65 @@
+// This file is dual licensed under the terms of the Apache License, Version
+// 2.0, and the BSD License. See the LICENSE file in the root of this
+// repository for complete details.
+
+/* Returns the value of the input with the most-significant-bit copied to all
+   of the bits. */
+static uint16_t Cryptography_DUPLICATE_MSB_TO_ALL(uint16_t a) {
+    return (1 - (a >> (sizeof(uint16_t) * 8 - 1))) - 1;
+}
+
+/* This returns 0xFFFF if a < b else 0x0000, but does so in a constant time
+   fashion */
+static uint16_t Cryptography_constant_time_lt(uint16_t a, uint16_t b) {
+    a -= b;
+    return Cryptography_DUPLICATE_MSB_TO_ALL(a);
+}
+
+uint8_t Cryptography_check_pkcs7_padding(const uint8_t *data,
+                                         uint16_t block_len) {
+    uint16_t i;
+    uint16_t pad_size = data[block_len - 1];
+    uint16_t mismatch = 0;
+    for (i = 0; i < block_len; i++) {
+        unsigned int mask = Cryptography_constant_time_lt(i, pad_size);
+        uint16_t b = data[block_len - 1 - i];
+        mismatch |= (mask & (pad_size ^ b));
+    }
+
+    /* Check to make sure the pad_size was within the valid range. */
+    mismatch |= ~Cryptography_constant_time_lt(0, pad_size);
+    mismatch |= Cryptography_constant_time_lt(block_len, pad_size);
+
+    /* Make sure any bits set are copied to the lowest bit */
+    mismatch |= mismatch >> 8;
+    mismatch |= mismatch >> 4;
+    mismatch |= mismatch >> 2;
+    mismatch |= mismatch >> 1;
+    /* Now check the low bit to see if it's set */
+    return (mismatch & 1) == 0;
+}
+
+uint8_t Cryptography_check_ansix923_padding(const uint8_t *data,
+                                            uint16_t block_len) {
+    uint16_t i;
+    uint16_t pad_size = data[block_len - 1];
+    uint16_t mismatch = 0;
+    /* Skip the first one with the pad size */
+    for (i = 1; i < block_len; i++) {
+        unsigned int mask = Cryptography_constant_time_lt(i, pad_size);
+        uint16_t b = data[block_len - 1 - i];
+        mismatch |= (mask & b);
+    }
+
+    /* Check to make sure the pad_size was within the valid range. */
+    mismatch |= ~Cryptography_constant_time_lt(0, pad_size);
+    mismatch |= Cryptography_constant_time_lt(block_len, pad_size);
+
+    /* Make sure any bits set are copied to the lowest bit */
+    mismatch |= mismatch >> 8;
+    mismatch |= mismatch >> 4;
+    mismatch |= mismatch >> 2;
+    mismatch |= mismatch >> 1;
+    /* Now check the low bit to see if it's set */
+    return (mismatch & 1) == 0;
+}
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/padding.h b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/padding.h
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/padding.h
@@ -0,0 +1,6 @@
+// This file is dual licensed under the terms of the Apache License, Version
+// 2.0, and the BSD License. See the LICENSE file in the root of this
+// repository for complete details.
+
+uint8_t Cryptography_check_pkcs7_padding(const uint8_t *, uint8_t);
+uint8_t Cryptography_check_ansix923_padding(const uint8_t *, uint8_t);
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/__init__.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/__init__.py
@@ -0,0 +1,5 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/aes.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/aes.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/aes.py
@@ -0,0 +1,50 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <openssl/aes.h>
+"""
+
+TYPES = """
+static const int Cryptography_HAS_AES_WRAP;
+static const int Cryptography_HAS_AES_CTR128_ENCRYPT;
+
+struct aes_key_st {
+    ...;
+};
+typedef struct aes_key_st AES_KEY;
+"""
+
+FUNCTIONS = """
+int AES_set_encrypt_key(const unsigned char *, const int, AES_KEY *);
+int AES_set_decrypt_key(const unsigned char *, const int, AES_KEY *);
+
+int AES_wrap_key(AES_KEY *, const unsigned char *, unsigned char *,
+                 const unsigned char *, unsigned int);
+int AES_unwrap_key(AES_KEY *, const unsigned char *, unsigned char *,
+                   const unsigned char *, unsigned int);
+"""
+
+MACROS = """
+/* The ctr128_encrypt function is only useful in 1.0.0. We can use EVP for
+   this in 1.0.1+. */
+void AES_ctr128_encrypt(const unsigned char *, unsigned char *,
+                        size_t, const AES_KEY *, unsigned char[],
+                        unsigned char[], unsigned int *);
+"""
+
+CUSTOMIZATIONS = """
+static const long Cryptography_HAS_AES_WRAP = 1;
+#if CRYPTOGRAPHY_OPENSSL_110_OR_GREATER && !defined(LIBRESSL_VERSION_NUMBER)
+static const int Cryptography_HAS_AES_CTR128_ENCRYPT = 0;
+void (*AES_ctr128_encrypt)(const unsigned char *, unsigned char *,
+                           size_t, const AES_KEY *,
+                           unsigned char[], unsigned char[],
+                           unsigned int *) = NULL;
+#else
+static const int Cryptography_HAS_AES_CTR128_ENCRYPT = 1;
+#endif
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/asn1.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/asn1.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/asn1.py
@@ -0,0 +1,164 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <openssl/asn1.h>
+"""
+
+TYPES = """
+typedef int... time_t;
+
+typedef int ASN1_BOOLEAN;
+typedef ... ASN1_INTEGER;
+
+struct asn1_string_st {
+    int length;
+    int type;
+    unsigned char *data;
+    long flags;
+};
+
+typedef struct asn1_string_st ASN1_OCTET_STRING;
+typedef struct asn1_string_st ASN1_IA5STRING;
+typedef struct asn1_string_st ASN1_BIT_STRING;
+typedef struct asn1_string_st ASN1_TIME;
+typedef ... ASN1_OBJECT;
+typedef struct asn1_string_st ASN1_STRING;
+typedef struct asn1_string_st ASN1_UTF8STRING;
+typedef ... ASN1_TYPE;
+typedef ... ASN1_GENERALIZEDTIME;
+typedef ... ASN1_ENUMERATED;
+typedef ... ASN1_ITEM;
+typedef ... ASN1_VALUE;
+
+typedef ... ASN1_ITEM_EXP;
+
+typedef ... ASN1_UTCTIME;
+
+static const int V_ASN1_GENERALIZEDTIME;
+
+static const int MBSTRING_FLAG;
+static const int MBSTRING_ASC;
+static const int MBSTRING_BMP;
+static const int MBSTRING_UTF8;
+static const int MBSTRING_UNIV;
+"""
+
+FUNCTIONS = """
+ASN1_OBJECT *ASN1_OBJECT_new(void);
+void ASN1_OBJECT_free(ASN1_OBJECT *);
+
+/*  ASN1 OBJECT IDENTIFIER */
+ASN1_OBJECT *d2i_ASN1_OBJECT(ASN1_OBJECT **, const unsigned char **, long);
+
+/*  ASN1 STRING */
+ASN1_STRING *ASN1_STRING_new(void);
+ASN1_STRING *ASN1_STRING_type_new(int);
+void ASN1_STRING_free(ASN1_STRING *);
+unsigned char *ASN1_STRING_data(ASN1_STRING *);
+int ASN1_STRING_set(ASN1_STRING *, const void *, int);
+
+/*  ASN1 OCTET STRING */
+ASN1_OCTET_STRING *ASN1_OCTET_STRING_new(void);
+void ASN1_OCTET_STRING_free(ASN1_OCTET_STRING *);
+int ASN1_OCTET_STRING_set(ASN1_OCTET_STRING *, const unsigned char *, int);
+
+/* ASN1 IA5STRING */
+ASN1_IA5STRING *ASN1_IA5STRING_new(void);
+
+/*  ASN1 INTEGER */
+ASN1_INTEGER *ASN1_INTEGER_new(void);
+void ASN1_INTEGER_free(ASN1_INTEGER *);
+int ASN1_INTEGER_set(ASN1_INTEGER *, long);
+
+/*  ASN1 TIME */
+ASN1_TIME *ASN1_TIME_new(void);
+void ASN1_TIME_free(ASN1_TIME *);
+ASN1_GENERALIZEDTIME *ASN1_TIME_to_generalizedtime(ASN1_TIME *,
+                                                   ASN1_GENERALIZEDTIME **);
+ASN1_TIME *ASN1_TIME_set(ASN1_TIME *, time_t);
+
+/*  ASN1 UTCTIME */
+ASN1_UTCTIME *ASN1_UTCTIME_new(void);
+void ASN1_UTCTIME_free(ASN1_UTCTIME *);
+int ASN1_UTCTIME_cmp_time_t(const ASN1_UTCTIME *, time_t);
+ASN1_UTCTIME *ASN1_UTCTIME_set(ASN1_UTCTIME *, time_t);
+
+/*  ASN1 GENERALIZEDTIME */
+int ASN1_GENERALIZEDTIME_set_string(ASN1_GENERALIZEDTIME *, const char *);
+ASN1_GENERALIZEDTIME *ASN1_GENERALIZEDTIME_set(ASN1_GENERALIZEDTIME *, time_t);
+void ASN1_GENERALIZEDTIME_free(ASN1_GENERALIZEDTIME *);
+int i2d_ASN1_GENERALIZEDTIME(ASN1_GENERALIZEDTIME *, unsigned char **);
+
+/*  ASN1 ENUMERATED */
+ASN1_ENUMERATED *ASN1_ENUMERATED_new(void);
+void ASN1_ENUMERATED_free(ASN1_ENUMERATED *);
+int ASN1_ENUMERATED_set(ASN1_ENUMERATED *, long);
+int i2d_ASN1_ENUMERATED(ASN1_ENUMERATED *, unsigned char **);
+
+ASN1_VALUE *ASN1_item_d2i(ASN1_VALUE **, const unsigned char **, long,
+                          const ASN1_ITEM *);
+int ASN1_BIT_STRING_set_bit(ASN1_BIT_STRING *, int, int);
+"""
+
+MACROS = """
+/* These became const ASN1_* in 1.1.0 */
+int i2d_ASN1_OBJECT(ASN1_OBJECT *, unsigned char **);
+int ASN1_STRING_type(ASN1_STRING *);
+int ASN1_STRING_to_UTF8(unsigned char **, ASN1_STRING *);
+long ASN1_ENUMERATED_get(ASN1_ENUMERATED *);
+int i2a_ASN1_INTEGER(BIO *, ASN1_INTEGER *);
+
+ASN1_UTF8STRING *ASN1_UTF8STRING_new(void);
+void ASN1_UTF8STRING_free(ASN1_UTF8STRING *);
+
+ASN1_BIT_STRING *ASN1_BIT_STRING_new(void);
+void ASN1_BIT_STRING_free(ASN1_BIT_STRING *);
+int i2d_ASN1_BIT_STRING(ASN1_BIT_STRING *, unsigned char **);
+int i2d_ASN1_OCTET_STRING(ASN1_OCTET_STRING *, unsigned char **);
+int i2d_ASN1_INTEGER(ASN1_INTEGER *, unsigned char **);
+/* This is not a macro, but is const on some versions of OpenSSL */
+int ASN1_BIT_STRING_get_bit(ASN1_BIT_STRING *, int);
+ASN1_TIME *M_ASN1_TIME_dup(void *);
+const ASN1_ITEM *ASN1_ITEM_ptr(ASN1_ITEM_EXP *);
+
+/* These aren't macros these arguments are all const X on openssl > 1.0.x */
+
+int ASN1_TIME_print(BIO *, ASN1_TIME *);
+int ASN1_STRING_length(ASN1_STRING *);
+ASN1_STRING *ASN1_STRING_dup(ASN1_STRING *);
+int ASN1_STRING_cmp(ASN1_STRING *, ASN1_STRING *);
+int ASN1_UTCTIME_print(BIO *, ASN1_UTCTIME *);
+
+ASN1_OCTET_STRING *ASN1_OCTET_STRING_dup(ASN1_OCTET_STRING *);
+int ASN1_OCTET_STRING_cmp(ASN1_OCTET_STRING *, ASN1_OCTET_STRING *);
+
+ASN1_INTEGER *ASN1_INTEGER_dup(ASN1_INTEGER *);
+int ASN1_INTEGER_cmp(ASN1_INTEGER *, ASN1_INTEGER *);
+long ASN1_INTEGER_get(ASN1_INTEGER *);
+
+BIGNUM *ASN1_INTEGER_to_BN(ASN1_INTEGER *, BIGNUM *);
+ASN1_INTEGER *BN_to_ASN1_INTEGER(BIGNUM *, ASN1_INTEGER *);
+
+/* These isn't a macro the arg is const on openssl 1.0.2+ */
+int ASN1_GENERALIZEDTIME_check(ASN1_GENERALIZEDTIME *);
+int ASN1_UTCTIME_check(ASN1_UTCTIME *);
+
+/* Not a macro, const on openssl 1.0 */
+int ASN1_STRING_set_default_mask_asc(char *);
+
+int i2d_ASN1_TYPE(ASN1_TYPE *, unsigned char **);
+ASN1_TYPE *d2i_ASN1_TYPE(ASN1_TYPE **, const unsigned char **, long);
+"""
+
+CUSTOMIZATIONS = """
+/* This macro is removed in 1.1.0. We re-add it if required to support
+   pyOpenSSL versions older than whatever resolves
+   https://github.com/pyca/pyopenssl/issues/431 */
+#if !defined(M_ASN1_TIME_dup)
+#define M_ASN1_TIME_dup(a) (ASN1_TIME *)ASN1_STRING_dup((const ASN1_STRING *)a)
+#endif
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/bignum.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/bignum.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/bignum.py
@@ -0,0 +1,88 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <openssl/bn.h>
+"""
+
+TYPES = """
+typedef ... BN_CTX;
+typedef ... BIGNUM;
+typedef int... BN_ULONG;
+"""
+
+FUNCTIONS = """
+BIGNUM *BN_new(void);
+void BN_free(BIGNUM *);
+
+BN_CTX *BN_CTX_new(void);
+void BN_CTX_free(BN_CTX *);
+
+void BN_CTX_start(BN_CTX *);
+BIGNUM *BN_CTX_get(BN_CTX *);
+void BN_CTX_end(BN_CTX *);
+
+BIGNUM *BN_copy(BIGNUM *, const BIGNUM *);
+BIGNUM *BN_dup(const BIGNUM *);
+
+int BN_set_word(BIGNUM *, BN_ULONG);
+BN_ULONG BN_get_word(const BIGNUM *);
+
+const BIGNUM *BN_value_one(void);
+
+char *BN_bn2hex(const BIGNUM *);
+int BN_hex2bn(BIGNUM **, const char *);
+int BN_dec2bn(BIGNUM **, const char *);
+
+int BN_bn2bin(const BIGNUM *, unsigned char *);
+BIGNUM *BN_bin2bn(const unsigned char *, int, BIGNUM *);
+
+int BN_num_bits(const BIGNUM *);
+
+int BN_cmp(const BIGNUM *, const BIGNUM *);
+int BN_add(BIGNUM *, const BIGNUM *, const BIGNUM *);
+int BN_sub(BIGNUM *, const BIGNUM *, const BIGNUM *);
+int BN_mul(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+int BN_sqr(BIGNUM *, const BIGNUM *, BN_CTX *);
+int BN_div(BIGNUM *, BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+int BN_nnmod(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+int BN_mod_add(BIGNUM *, const BIGNUM *, const BIGNUM *, const BIGNUM *,
+               BN_CTX *);
+int BN_mod_sub(BIGNUM *, const BIGNUM *, const BIGNUM *, const BIGNUM *,
+               BN_CTX *);
+int BN_mod_mul(BIGNUM *, const BIGNUM *, const BIGNUM *, const BIGNUM *,
+               BN_CTX *);
+int BN_mod_sqr(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+int BN_exp(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+int BN_mod_exp(BIGNUM *, const BIGNUM *, const BIGNUM *, const BIGNUM *,
+               BN_CTX *);
+int BN_gcd(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+BIGNUM *BN_mod_inverse(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+
+int BN_set_bit(BIGNUM *, int);
+int BN_clear_bit(BIGNUM *, int);
+
+int BN_is_bit_set(const BIGNUM *, int);
+
+int BN_mask_bits(BIGNUM *, int);
+"""
+
+MACROS = """
+int BN_num_bytes(const BIGNUM *);
+
+int BN_zero(BIGNUM *);
+int BN_one(BIGNUM *);
+int BN_mod(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+
+int BN_lshift(BIGNUM *, const BIGNUM *, int);
+int BN_lshift1(BIGNUM *, BIGNUM *);
+
+int BN_rshift(BIGNUM *, BIGNUM *, int);
+int BN_rshift1(BIGNUM *, BIGNUM *);
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/bio.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/bio.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/bio.py
@@ -0,0 +1,147 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <openssl/bio.h>

From pypy.commits at gmail.com  Sat May 25 16:14:20 2019
From: pypy.commits at gmail.com (mattip)
Date: Sat, 25 May 2019 13:14:20 -0700 (PDT)
Subject: [pypy-commit] pypy default: change error type
Message-ID: <5ce9a21c.1c69fb81.571b1.5a4a@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r96682:144422c7850b
Date: 2019-05-25 22:29 +0300
http://bitbucket.org/pypy/pypy/changeset/144422c7850b/

Log:	change error type

diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -333,7 +333,7 @@
                 sockstate = SOCKET_OPERATION_OK
 
             if sockstate == SOCKET_HAS_TIMED_OUT:
-                raise socket.timeout("The handshake operation timed out")
+                raise SSLError("The handshake operation timed out")
             elif sockstate == SOCKET_HAS_BEEN_CLOSED:
                 raise SSLError("Underlying socket has been closed.")
             elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT:

From pypy.commits at gmail.com  Sat May 25 16:14:22 2019
From: pypy.commits at gmail.com (mattip)
Date: Sat, 25 May 2019 13:14:22 -0700 (PDT)
Subject: [pypy-commit] pypy default: disable protocols not available on
 python2
Message-ID: <5ce9a21e.1c69fb81.22500.1d89@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r96683:c8dfd959151f
Date: 2019-05-25 23:07 +0300
http://bitbucket.org/pypy/pypy/changeset/c8dfd959151f/

Log:	disable protocols not available on python2

diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -87,8 +87,8 @@
     PROTOCOL_TLSv1 = 3
     PROTOCOL_TLSv1_1 = 4
     PROTOCOL_TLSv1_2 = 5
-PROTOCOL_TLS_CLIENT = 0x10
-PROTOCOL_TLS_SERVER = 0x11
+# PROTOCOL_TLS_CLIENT = 0x10
+# PROTOCOL_TLS_SERVER = 0x11
 
 _PROTOCOL_NAMES = (name for name in dir(lib) if name.startswith('PROTOCOL_'))
 
@@ -777,10 +777,10 @@
             method = lib.SSLv2_method()
         elif protocol == PROTOCOL_SSLv23:
             method = lib.SSLv23_method()
-        elif protocol == PROTOCOL_TLS_CLIENT:
-            method = lib.SSLv23_client_method()
-        elif protocol == PROTOCOL_TLS_SERVER:
-            method = lib.SSLv23_server_method()
+        # elif protocol == PROTOCOL_TLS_CLIENT:
+        #     method = lib.SSLv23_client_method()
+        # elif protocol == PROTOCOL_TLS_SERVER:
+        #     method = lib.SSLv23_server_method()
         else:
             raise ValueError("invalid protocol version")
 
@@ -791,7 +791,7 @@
 
         # Don't check host name by default
         self._check_hostname = False
-        if protocol == PROTOCOL_TLS_CLIENT:
+        if 0 and protocol == PROTOCOL_TLS_CLIENT:
             self._check_hostname = True
             self.verify_mode = CERT_REQUIRED
         else:

From pypy.commits at gmail.com  Sat May 25 16:55:48 2019
From: pypy.commits at gmail.com (mattip)
Date: Sat, 25 May 2019 13:55:48 -0700 (PDT)
Subject: [pypy-commit] pypy default: cleanup unused import
Message-ID: <5ce9abd4.1c69fb81.2aee3.321b@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r96684:6b079b8365ca
Date: 2019-05-25 23:55 +0300
http://bitbucket.org/pypy/pypy/changeset/6b079b8365ca/

Log:	cleanup unused import

diff --git a/lib-python/2.7/ctypes/test/test_byteswap.py b/lib-python/2.7/ctypes/test/test_byteswap.py
--- a/lib-python/2.7/ctypes/test/test_byteswap.py
+++ b/lib-python/2.7/ctypes/test/test_byteswap.py
@@ -2,7 +2,6 @@
 from binascii import hexlify
 
 from ctypes import *
-from ctypes.test import xfail
 
 def bin(s):
     return hexlify(memoryview(s)).upper()

From pypy.commits at gmail.com  Sat May 25 17:56:51 2019
From: pypy.commits at gmail.com (mattip)
Date: Sat, 25 May 2019 14:56:51 -0700 (PDT)
Subject: [pypy-commit] pypy stdlib-2.7.16: partially update _cffi_ssl to get
 TLSv1_3
Message-ID: <5ce9ba23.1c69fb81.62076.6624@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: stdlib-2.7.16
Changeset: r96685:580e7f2d8380
Date: 2019-05-26 00:56 +0300
http://bitbucket.org/pypy/pypy/changeset/580e7f2d8380/

Log:	partially update _cffi_ssl to get TLSv1_3

diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
@@ -17,6 +17,7 @@
 static const long Cryptography_HAS_SSL3_METHOD;
 static const long Cryptography_HAS_TLSv1_1;
 static const long Cryptography_HAS_TLSv1_2;
+static const long Cryptography_HAS_TLSv1_3;
 static const long Cryptography_HAS_SECURE_RENEGOTIATION;
 static const long Cryptography_HAS_COMPRESSION;
 static const long Cryptography_HAS_TLSEXT_STATUS_REQ_CB;
@@ -39,13 +40,12 @@
  * supported
  */
 static const long Cryptography_HAS_OP_NO_COMPRESSION;
-
 static const long Cryptography_HAS_SSL_OP_MSIE_SSLV2_RSA_PADDING;
 static const long Cryptography_HAS_SSL_SET_SSL_CTX;
 static const long Cryptography_HAS_SSL_OP_NO_TICKET;
 static const long Cryptography_HAS_NETBSD_D1_METH;
+static const long Cryptography_HAS_ALPN;
 static const long Cryptography_HAS_NEXTPROTONEG;
-static const long Cryptography_HAS_ALPN;
 static const long Cryptography_HAS_SET_CERT_CB;
 
 static const long SSL_FILETYPE_PEM;
@@ -65,13 +65,13 @@
 static const long SSL_OP_NO_TLSv1;
 static const long SSL_OP_NO_TLSv1_1;
 static const long SSL_OP_NO_TLSv1_2;
+static const long SSL_OP_NO_TLSv1_3;
 static const long SSL_OP_NO_COMPRESSION;
 static const long SSL_OP_SINGLE_DH_USE;
 static const long SSL_OP_EPHEMERAL_RSA;
 static const long SSL_OP_MICROSOFT_SESS_ID_BUG;
 static const long SSL_OP_NETSCAPE_CHALLENGE_BUG;
 static const long SSL_OP_NETSCAPE_REUSE_CIPHER_CHANGE_BUG;
-static const long SSL_OP_NO_SSLv2;
 static const long SSL_OP_SSLREF2_REUSE_CERT_TYPE_BUG;
 static const long SSL_OP_MICROSOFT_BIG_SSLV3_BUFFER;
 static const long SSL_OP_MSIE_SSLV2_RSA_PADDING;
@@ -356,7 +356,7 @@
 void SSL_set_tlsext_host_name(SSL *, char *);
 void SSL_CTX_set_tlsext_servername_callback(
     SSL_CTX *,
-    int (*)(const SSL *, int *, void *));
+    int (*)(SSL *, int *, void *));
 void SSL_CTX_set_tlsext_servername_arg(
     SSL_CTX *, void *);
 
@@ -514,10 +514,12 @@
     memcpy(out, session->master_key, outlen);
     return outlen;
 }
-
-int SSL_SESSION_has_ticket(const SSL_SESSION *s) {
+/* from ssl/ssl_sess.c */
+int SSL_SESSION_has_ticket(const SSL_SESSION *s)
+{
     return (s->tlsext_ticklen > 0) ? 1 : 0;
 }
+/* from ssl/ssl_sess.c */
 unsigned long SSL_SESSION_get_ticket_lifetime_hint(const SSL_SESSION *s)
 {
     return s->tlsext_tick_lifetime_hint;
@@ -729,4 +731,19 @@
 static const long OPENSSL_NPN_NEGOTIATED = -1;
 static const long Cryptography_HAS_NPN_NEGOTIATED = 0;
 #endif
+#if CRYPTOGRAPHY_OPENSSL_LESS_THAN_111
+static const long Cryptography_HAS_TLSv1_3 = 0;
+static const long SSL_OP_NO_TLSv1_3 = 0;
+static const long SSL_VERIFY_POST_HANDSHAKE = 0;
+int (*SSL_CTX_set_ciphersuites)(SSL_CTX *, const char *) = NULL;
+int (*SSL_verify_client_post_handshake)(SSL *) = NULL;
+void (*SSL_CTX_set_post_handshake_auth)(SSL_CTX *, int) = NULL;
+void (*SSL_set_post_handshake_auth)(SSL *, int) = NULL;
+uint32_t (*SSL_SESSION_get_max_early_data)(const SSL_SESSION *) = NULL;
+int (*SSL_write_early_data)(SSL *, const void *, size_t, size_t *) = NULL;
+int (*SSL_read_early_data)(SSL *, void *, size_t, size_t *) = NULL;
+int (*SSL_CTX_set_max_early_data)(SSL_CTX *, uint32_t) = NULL;
+#else
+static const long Cryptography_HAS_TLSv1_3 = 1;
+#endif
 """
diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -87,6 +87,10 @@
     PROTOCOL_TLSv1 = 3
     PROTOCOL_TLSv1_1 = 4
     PROTOCOL_TLSv1_2 = 5
+if lib.Cryptography_HAS_TLSv1_3:
+    HAS_TLSv1_3 = True
+else:
+    HAS_TLSv1_3 = False
 PROTOCOL_TLS_CLIENT = 0x10
 PROTOCOL_TLS_SERVER = 0x11
 
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -26,7 +26,7 @@
 #define PY_RELEASE_SERIAL	0
 
 /* Version as a string */
-#define PY_VERSION		"2.7.13"
+#define PY_VERSION		"2.7.16"
 
 /* PyPy version as a string: make sure to keep this in sync with:
  *     module/sys/version.py

From pypy.commits at gmail.com  Sun May 26 07:26:27 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 26 May 2019 04:26:27 -0700 (PDT)
Subject: [pypy-commit] pypy default: set owner attribute,
 fix test for more modern OpenSSL
Message-ID: <5cea77e3.1c69fb81.da87e.e41b@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r96686:a6c18dc8a3c6
Date: 2019-05-26 13:57 +0300
http://bitbucket.org/pypy/pypy/changeset/a6c18dc8a3c6/

Log:	set owner attribute, fix test for more modern OpenSSL

diff --git a/lib-python/2.7/test/test_ssl.py b/lib-python/2.7/test/test_ssl.py
--- a/lib-python/2.7/test/test_ssl.py
+++ b/lib-python/2.7/test/test_ssl.py
@@ -770,6 +770,7 @@
         default = (ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3)
         if not IS_LIBRESSL and ssl.OPENSSL_VERSION_INFO >= (1, 1, 0):
             default |= ssl.OP_NO_COMPRESSION
+            default |= ssl.OP_ENABLE_MIDDLEBOX_COMPAT
         self.assertEqual(default, ctx.options)
         ctx.options |= ssl.OP_NO_TLSv1
         self.assertEqual(default | ssl.OP_NO_TLSv1, ctx.options)
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
@@ -73,6 +73,7 @@
 static const long SSL_OP_NETSCAPE_REUSE_CIPHER_CHANGE_BUG;
 static const long SSL_OP_NO_SSLv2;
 static const long SSL_OP_SSLREF2_REUSE_CERT_TYPE_BUG;
+static const long SSL_OP_ENABLE_MIDDLEBOX_COMPAT;
 static const long SSL_OP_MICROSOFT_BIG_SSLV3_BUFFER;
 static const long SSL_OP_MSIE_SSLV2_RSA_PADDING;
 static const long SSL_OP_SSLEAY_080_CLIENT_DH_BUG;
diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -215,6 +215,7 @@
     def _new__ssl_socket(sslctx, sock, socket_type, server_hostname, ssl_sock):
         self = _SSLSocket(sslctx)
         ctx = sslctx.ctx
+        self.owner = ssl_sock  # weakref
 
         if server_hostname:
             if isinstance(server_hostname, unicode):
@@ -285,7 +286,8 @@
     def owner(self, value):
         if value is None:
             self._owner = None
-        self._owner = weakref.ref(value)
+        else:
+            self._owner = weakref.ref(value)
 
     @property
     def context(self):
@@ -807,7 +809,7 @@
         # Minimal security flags for server and client side context.
         # Client sockets ignore server-side parameters.
         options |= lib.SSL_OP_NO_COMPRESSION
-        options |= lib.SSL_OP_CIPHER_SERVER_PREFERENCE
+        # options |= lib.SSL_OP_CIPHER_SERVER_PREFERENCE
         options |= lib.SSL_OP_SINGLE_DH_USE
         options |= lib.SSL_OP_SINGLE_ECDH_USE
         lib.SSL_CTX_set_options(self.ctx, options)

From pypy.commits at gmail.com  Sun May 26 09:09:39 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 26 May 2019 06:09:39 -0700 (PDT)
Subject: [pypy-commit] pypy default: handle old OpenSSL without
 SSL_OP_ENABLE_MIDDLEBOX_COMPAT
Message-ID: <5cea9013.1c69fb81.22ab3.706c@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r96687:6658d2817ce4
Date: 2019-05-26 16:07 +0300
http://bitbucket.org/pypy/pypy/changeset/6658d2817ce4/

Log:	handle old OpenSSL without SSL_OP_ENABLE_MIDDLEBOX_COMPAT

diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
@@ -40,6 +40,11 @@
  */
 static const long Cryptography_HAS_OP_NO_COMPRESSION;
 
+/* Internally invented symbol to tell us if SSL_OP_ENABLE_MIDDLEBOX_COMPAT is
+ * supported
+ */
+static const long Cryptography_HAS_OP_ENABLE_MIDDLEBOX_COMPAT;
+
 static const long Cryptography_HAS_SSL_OP_MSIE_SSLV2_RSA_PADDING;
 static const long Cryptography_HAS_SSL_SET_SSL_CTX;
 static const long Cryptography_HAS_SSL_OP_NO_TICKET;
@@ -561,6 +566,13 @@
 const long SSL_OP_NO_COMPRESSION = 0;
 #endif
 
+#ifdef SSL_OP_ENABLE_MIDDLEBOX_COMPAT
+static const long Cryptography_HAS_OP_ENABLE_MIDDLEBOX_COMPAT = 1;
+#else
+static const long Cryptography_HAS_OP_ENABLE_MIDDLEBOX_COMPAT = 0;
+const long SSL_OP_ENABLE_MIDDLEBOX_COMPAT = 0;
+#endif
+
 #ifdef SSL_OP_NO_TLSv1_1
 static const long Cryptography_HAS_TLSv1_1 = 1;
 #else

From pypy.commits at gmail.com  Sun May 26 11:24:34 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 26 May 2019 08:24:34 -0700 (PDT)
Subject: [pypy-commit] pypy default: disable flaky test
Message-ID: <5ceaafb2.1c69fb81.571b1.e124@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r96688:937db12c1a36
Date: 2019-05-26 18:23 +0300
http://bitbucket.org/pypy/pypy/changeset/937db12c1a36/

Log:	disable flaky test

diff --git a/lib-python/2.7/test/test_ssl.py b/lib-python/2.7/test/test_ssl.py
--- a/lib-python/2.7/test/test_ssl.py
+++ b/lib-python/2.7/test/test_ssl.py
@@ -764,7 +764,11 @@
             ctx.set_ciphers("^$:,;?*'dorothyx")
 
     @skip_if_broken_ubuntu_ssl
-    def test_options(self):
+    def _test_options(self):
+        '''
+        Disable this test, it is too flaky. Different platforms define
+        different defaults
+        '''
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
         # OP_ALL | OP_NO_SSLv2 | OP_NO_SSLv3 is the default value
         default = (ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3)

From pypy.commits at gmail.com  Sun May 26 15:37:45 2019
From: pypy.commits at gmail.com (andrewjlawrence)
Date: Sun, 26 May 2019 12:37:45 -0700 (PDT)
Subject: [pypy-commit] pypy winmultiprocessing: Add app level sharelocal
 test.
Message-ID: <5ceaeb09.1c69fb81.379ac.d9bf@mx.google.com>

Author: andrewjlawrence
Branch: winmultiprocessing
Changeset: r96689:010688999eca
Date: 2019-05-26 20:30 +0100
http://bitbucket.org/pypy/pypy/changeset/010688999eca/

Log:	Add app level sharelocal test.

diff --git a/lib_pypy/_winapi.py b/lib_pypy/_winapi.py
--- a/lib_pypy/_winapi.py
+++ b/lib_pypy/_winapi.py
@@ -114,7 +114,6 @@
                 #raise RuntimeError('deleting an overlapped struct with a pending operation not supported')
         CloseHandle(_int2handle(self.overlapped[0].hEvent))
         _kernel32.SetLastError(err)
-        err = _kernel32.GetLastError()
 
     @property
     def event(self):
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -574,13 +574,26 @@
         s = _socket.socket()
         raises(ValueError, s.ioctl, -1, None)
         s.ioctl(_socket.SIO_KEEPALIVE_VALS, (1, 100, 100))
-		
+
     def test_socket_sharelocal(self):
         import _socket, sys, os
         if sys.platform != 'win32':
             skip("win32 only")
         assert hasattr(_socket.socket, 'share')
-        s = _socket.socket()
+        s = _socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        s.listen()
+        data = s.share(os.getpid())
+        s2 = socket.fromshare(data)
+        try:
+            assert s.gettimeout() == s2.gettimeout()
+            assert s.family == s2.family
+            assert s.type == s2.type
+            if s.proto != 0:
+                assert s.proto == s2.proto
+        finally:
+            s.close()
+            s2.close()
+
 
     def test_dup(self):
         import _socket as socket, os

From pypy.commits at gmail.com  Sun May 26 15:37:49 2019
From: pypy.commits at gmail.com (andrewjlawrence)
Date: Sun, 26 May 2019 12:37:49 -0700 (PDT)
Subject: [pypy-commit] pypy winmultiprocessing: Merge py3.6 branch
Message-ID: <5ceaeb0d.1c69fb81.d9af7.e41d@mx.google.com>

Author: andrewjlawrence
Branch: winmultiprocessing
Changeset: r96690:5833cf4a4240
Date: 2019-05-26 20:36 +0100
http://bitbucket.org/pypy/pypy/changeset/5833cf4a4240/

Log:	Merge py3.6 branch

diff --git a/lib-python/2.7/test/capath/efa5f9c3.0 b/lib-python/2.7/test/capath/efa5f9c3.0
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/capath/efa5f9c3.0
@@ -0,0 +1,34 @@
+-----BEGIN CERTIFICATE-----
+MIIF9zCCA9+gAwIBAgIUH98b4Fw/DyugC9cV7VK7ZODzHsIwDQYJKoZIhvcNAQEL
+BQAwgYoxCzAJBgNVBAYTAlhZMRcwFQYDVQQIDA5DYXN0bGUgQW50aHJheDEYMBYG
+A1UEBwwPQXJndW1lbnQgQ2xpbmljMSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUg
+Rm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0aG9udGVzdC5uZXQw
+HhcNMTkwNTA4MDEwMjQzWhcNMjcwNzI0MDEwMjQzWjCBijELMAkGA1UEBhMCWFkx
+FzAVBgNVBAgMDkNhc3RsZSBBbnRocmF4MRgwFgYDVQQHDA9Bcmd1bWVudCBDbGlu
+aWMxIzAhBgNVBAoMGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMSMwIQYDVQQD
+DBpzZWxmLXNpZ25lZC5weXRob250ZXN0Lm5ldDCCAiIwDQYJKoZIhvcNAQEBBQAD
+ggIPADCCAgoCggIBAMKdJlyCThkahwoBb7pl5q64Pe9Fn5jrIvzsveHTc97TpjV2
+RLfICnXKrltPk/ohkVl6K5SUZQZwMVzFubkyxE0nZPHYHlpiKWQxbsYVkYv01rix
+IFdLvaxxbGYke2jwQao31s4o61AdlsfK1SdpHQUynBBMssqI3SB4XPmcA7e+wEEx
+jxjVish4ixA1vuIZOx8yibu+CFCf/geEjoBMF3QPdzULzlrCSw8k/45iZCSoNbvK
+DoL4TVV07PHOxpheDh8ZQmepGvU6pVqhb9m4lgmV0OGWHgozd5Ur9CbTVDmxIEz3
+TSoRtNJK7qtyZdGNqwjksQxgZTjM/d/Lm/BJG99AiOmYOjsl9gbQMZgvQmMAtUsI
+aMJnQuZ6R+KEpW/TR5qSKLWZSG45z/op+tzI2m+cE6HwTRVAWbcuJxcAA55MZjqU
+OOOu3BBYMjS5nf2sQ9uoXsVBFH7i0mQqoW1SLzr9opI8KsWwFxQmO2vBxWYaN+lH
+OmwBZBwyODIsmI1YGXmTp09NxRYz3Qe5GCgFzYowpMrcxUC24iduIdMwwhRM7rKg
+7GtIWMSrFfuI1XCLRmSlhDbhNN6fVg2f8Bo9PdH9ihiIyxSrc+FOUasUYCCJvlSZ
+8hFUlLvcmrZlWuazohm0lsXuMK1JflmQr/DA/uXxP9xzFfRy+RU3jDyxJbRHAgMB
+AAGjUzBRMB0GA1UdDgQWBBSQJyxiPMRK01i+0BsV9zUwDiBaHzAfBgNVHSMEGDAW
+gBSQJyxiPMRK01i+0BsV9zUwDiBaHzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3
+DQEBCwUAA4ICAQCR+7a7N/m+WLkxPPIA/CB4MOr2Uf8ixTv435Nyv6rXOun0+lTP
+ExSZ0uYQ+L0WylItI3cQHULldDueD+s8TGzxf5woaLKf6tqyr0NYhKs+UeNEzDnN
+9PHQIhX0SZw3XyXGUgPNBfRCg2ZDdtMMdOU4XlQN/IN/9hbYTrueyY7eXq9hmtI9
+1srftAMqr9SR1JP7aHI6DVgrEsZVMTDnfT8WmLSGLlY1HmGfdEn1Ip5sbo9uSkiH
+AEPgPfjYIvR5LqTOMn4KsrlZyBbFIDh9Sl99M1kZzgH6zUGVLCDg1y6Cms69fx/e
+W1HoIeVkY4b4TY7Bk7JsqyNhIuqu7ARaxkdaZWhYaA2YyknwANdFfNpfH+elCLIk
+BUt5S3f4i7DaUePTvKukCZiCq4Oyln7RcOn5If73wCeLB/ZM9Ei1HforyLWP1CN8
+XLfpHaoeoPSWIveI0XHUl65LsPN2UbMbul/F23hwl+h8+BLmyAS680Yhn4zEN6Ku
+B7Po90HoFa1Du3bmx4jsN73UkT/dwMTi6K072FbipnC1904oGlWmLwvAHvrtxxmL
+Pl3pvEaZIu8wa/PNF6Y7J7VIewikIJq6Ta6FrWeFfzMWOj2qA1ZZi6fUaDSNYvuV
+J5quYKCc/O+I/yDDf8wyBbZ/gvUXzUHTMYGG+bFrn1p7XDbYYeEJ6R/xEg==
+-----END CERTIFICATE-----
diff --git a/lib-python/3/datetime.py b/lib-python/3/datetime.py
--- a/lib-python/3/datetime.py
+++ b/lib-python/3/datetime.py
@@ -6,6 +6,7 @@
 
 import time as _time
 import math as _math
+import sys
 
 # for cpyext, use these as base classes
 from __pypy__._pypydatetime import dateinterop, deltainterop, timeinterop
@@ -1379,7 +1380,8 @@
 
     def __new__(cls, year, month=None, day=None, hour=0, minute=0, second=0,
                 microsecond=0, tzinfo=None, *, fold=0):
-        if isinstance(year, bytes) and len(year) == 10 and 1 <= year[2]&0x7F <= 12:
+        if (isinstance(year, (bytes, str)) and len(year) == 10 and
+            1 <= ord(year[2:3])&0x7F <= 12):
             # Pickle support
             self = dateinterop.__new__(cls)
             self.__setstate(year, month)
@@ -1456,6 +1458,14 @@
             # 23 hours at 1969-09-30 13:00:00 in Kwajalein.
             # Let's probe 24 hours in the past to detect a transition:
             max_fold_seconds = 24 * 3600
+
+            # On Windows localtime_s throws an OSError for negative values,
+            # thus we can't perform fold detection for values of time less
+            # than the max time fold. See comments in _datetimemodule's
+            # version of this method for more details.
+            if t < max_fold_seconds and sys.platform.startswith("win"):
+                return result
+
             y, m, d, hh, mm, ss = converter(t - max_fold_seconds)[:6]
             probe1 = cls(y, m, d, hh, mm, ss, us, tz)
             trans = result - probe1 - timedelta(0, max_fold_seconds)
diff --git a/lib-python/3/sysconfig.py b/lib-python/3/sysconfig.py
--- a/lib-python/3/sysconfig.py
+++ b/lib-python/3/sysconfig.py
@@ -563,6 +563,7 @@
             _CONFIG_VARS['abiflags'] = ''
         _CONFIG_VARS['implementation'] = _get_implementation()
         _CONFIG_VARS['implementation_lower'] = _get_implementation().lower()
+        _CONFIG_VARS['LIBRARY'] = ''
 
         if os.name == 'nt':
             _init_non_posix(_CONFIG_VARS)
diff --git a/lib_pypy/_cffi_ssl/.gitignore b/lib_pypy/_cffi_ssl/.gitignore
deleted file mode 100644
--- a/lib_pypy/_cffi_ssl/.gitignore
+++ /dev/null
@@ -1,95 +0,0 @@
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-env/
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-*.egg-info/
-.installed.cfg
-*.egg
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*,cover
-.hypothesis/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# pyenv
-.python-version
-
-# celery beat schedule file
-celerybeat-schedule
-
-# dotenv
-.env
-
-# virtualenv
-.venv/
-venv/
-ENV/
-
-# Spyder project settings
-.spyderproject
-
-# Rope project settings
-.ropeproject
-
-# Vim
-
-*.swp
-*.swo
diff --git a/lib_pypy/_cffi_ssl/__init__.py b/lib_pypy/_cffi_ssl/__init__.py
new file mode 100644
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
@@ -71,6 +71,7 @@
 static const long SSL_OP_MICROSOFT_SESS_ID_BUG;
 static const long SSL_OP_NETSCAPE_CHALLENGE_BUG;
 static const long SSL_OP_NETSCAPE_REUSE_CIPHER_CHANGE_BUG;
+static const long SSL_OP_NO_SSLv2;
 static const long SSL_OP_SSLREF2_REUSE_CERT_TYPE_BUG;
 static const long SSL_OP_MICROSOFT_BIG_SSLV3_BUFFER;
 static const long SSL_OP_MSIE_SSLV2_RSA_PADDING;
diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -1,7 +1,6 @@
 import sys
 import time
 import _thread
-import socket
 import weakref
 from _pypy_openssl import ffi
 from _pypy_openssl import lib
@@ -70,6 +69,7 @@
             globals()[name[4:]] = getattr(lib, name)
 
 OP_ALL = lib.SSL_OP_ALL & ~lib.SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS
+OP_NO_SSLv2 = lib.SSL_OP_NO_SSLv2
 
 SSL_CLIENT = 0
 SSL_SERVER = 1
@@ -78,7 +78,8 @@
 
 if lib.Cryptography_HAS_SSL2:
     PROTOCOL_SSLv2  = 0
-PROTOCOL_SSLv3  = 1
+if lib.Cryptography_HAS_SSL3_METHOD:
+    PROTOCOL_SSLv3  = 1
 PROTOCOL_SSLv23 = 2
 PROTOCOL_TLS    = PROTOCOL_SSLv23
 PROTOCOL_TLSv1    = 3
@@ -310,6 +311,9 @@
         return self.socket_type == SSL_SERVER
 
     def do_handshake(self):
+        # delay to prevent circular imports
+        import socket
+
         sock = self.get_socket_or_connection_gone()
         ssl = self.ssl
         timeout = _socket_timeout(sock)
@@ -381,6 +385,9 @@
                 return _decode_certificate(self.peer_cert)
 
     def write(self, bytestring):
+        # delay to prevent circular imports
+        import socket
+
         deadline = 0
         b = _str_to_ffi_buffer(bytestring)
         sock = self.get_socket_or_connection_gone()
@@ -439,6 +446,9 @@
             raise pyssl_error(self, length)
 
     def read(self, length, buffer_into=None):
+        # delay to prevent circular imports
+        import socket
+
         ssl = self.ssl
 
         if length < 0 and buffer_into is None:
@@ -579,6 +589,9 @@
         return sock
 
     def shutdown(self):
+        # delay to prevent circular imports
+        import socket
+
         sock = self.get_socket_or_None()
         nonblocking = False
         ssl = self.ssl
@@ -804,7 +817,7 @@
             method = lib.TLSv1_1_method()
         elif lib.Cryptography_HAS_TLSv1_2 and protocol == PROTOCOL_TLSv1_2 :
             method = lib.TLSv1_2_method()
-        elif protocol == PROTOCOL_SSLv3 and lib.Cryptography_HAS_SSL3_METHOD:
+        elif lib.Cryptography_HAS_SSL3_METHOD and protocol == PROTOCOL_SSLv3:
             method = lib.SSLv3_method()
         elif lib.Cryptography_HAS_SSL2 and protocol == PROTOCOL_SSLv2:
             method = lib.SSLv2_method()
@@ -835,7 +848,7 @@
         options = lib.SSL_OP_ALL & ~lib.SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS
         if not lib.Cryptography_HAS_SSL2 or protocol != PROTOCOL_SSLv2:
             options |= lib.SSL_OP_NO_SSLv2
-        if protocol != PROTOCOL_SSLv3:
+        if not lib.Cryptography_HAS_SSL3_METHOD or protocol != PROTOCOL_SSLv3:
             options |= lib.SSL_OP_NO_SSLv3
         # Minimal security flags for server and client side context.
         # Client sockets ignore server-side parameters.
diff --git a/lib_pypy/_cffi_ssl/_stdssl/utility.py b/lib_pypy/_cffi_ssl/_stdssl/utility.py
--- a/lib_pypy/_cffi_ssl/_stdssl/utility.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/utility.py
@@ -19,7 +19,7 @@
     elif isinstance(view, memoryview):
         # NOTE pypy limitation StringBuffer does not allow
         # to get a raw address to the string!
-        view = bytes(view)
+        view = view.tobytes()
     # dont call call ffi.from_buffer(bytes(view)), arguments
     # like ints/bools should result in a TypeError
     return ffi.from_buffer(view)
diff --git a/lib_pypy/_ssl/__init__.py b/lib_pypy/_ssl/__init__.py
--- a/lib_pypy/_ssl/__init__.py
+++ b/lib_pypy/_ssl/__init__.py
@@ -3,6 +3,8 @@
 from _cffi_ssl import _stdssl
 from _cffi_ssl._stdssl import *
 
+OP_SINGLE_DH_USE = lib.SSL_OP_SINGLE_DH_USE
+OP_SINGLE_ECDH_USE = lib.SSL_OP_SINGLE_ECDH_USE
 
 try: from __pypy__ import builtinify
 except ImportError: builtinify = lambda f: f
diff --git a/lib_pypy/_sysconfigdata.py b/lib_pypy/_sysconfigdata.py
--- a/lib_pypy/_sysconfigdata.py
+++ b/lib_pypy/_sysconfigdata.py
@@ -20,7 +20,7 @@
     'AR': "ar",
     'ARFLAGS': "rc",
     'EXE': "",
-    'LIBDIR': os.path.join(sys.prefix, 'lib'),
+    'LIBDIR': os.path.join(sys.prefix, 'bin'),
     'VERSION': sys.version[:3]
 }
 
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -113,42 +113,6 @@
     module_suggests["cpyext"].append(("translation.shared", True))
 
 
-# NOTE: this dictionary is not used any more
-module_import_dependencies = {
-    # no _rawffi if importing rpython.rlib.clibffi raises ImportError
-    # or CompilationError or py.test.skip.Exception
-    "_rawffi"   : ["rpython.rlib.clibffi"],
-
-    "zlib"      : ["rpython.rlib.rzlib"],
-    "bz2"       : ["pypy.module.bz2.interp_bz2"],
-    "pyexpat"   : ["pypy.module.pyexpat.interp_pyexpat"],
-    "_minimal_curses": ["pypy.module._minimal_curses.fficurses"],
-    "_continuation": ["rpython.rlib.rstacklet"],
-    "_vmprof"      : ["pypy.module._vmprof.interp_vmprof"],
-    "faulthandler" : ["pypy.module._vmprof.interp_vmprof"],
-    "_lzma"     : ["pypy.module._lzma.interp_lzma"],
-    }
-
-def get_module_validator(modname):
-    # NOTE: this function is not used any more
-    if modname in module_import_dependencies:
-        modlist = module_import_dependencies[modname]
-        def validator(config):
-            from rpython.rtyper.tool.rffi_platform import CompilationError
-            try:
-                for name in modlist:
-                    __import__(name)
-            except (ImportError, CompilationError, py.test.skip.Exception) as e:
-                errcls = e.__class__.__name__
-                raise Exception(
-                    "The module %r is disabled\n" % (modname,) +
-                    "because importing %s raised %s\n" % (name, errcls) +
-                    str(e))
-        return validator
-    else:
-        return None
-
-
 pypy_optiondescription = OptionDescription("objspace", "Object Space Options", [
     OptionDescription("usemodules", "Which Modules should be used", [
         BoolOption(modname, "use module %s" % (modname, ),
@@ -157,7 +121,7 @@
                    requires=module_dependencies.get(modname, []),
                    suggests=module_suggests.get(modname, []),
                    negation=modname not in essential_modules,
-                   ) #validator=get_module_validator(modname))
+                   )
         for modname in all_modules]),
 
     BoolOption("allworkingmodules", "use as many working modules as possible",
diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst
--- a/pypy/doc/how-to-release.rst
+++ b/pypy/doc/how-to-release.rst
@@ -8,6 +8,12 @@
 a branch named like release-pypy3.5-v2.x or release-pypy3.5-v4.x, and each
 release is tagged, for instance release-pypy3.5-v4.0.1. 
 
+The release version number should be bumped. A micro release increment means
+there were no changes that justify rebuilding c-extension wheels, since
+the wheels are marked with only major.minor version numbers. It is ofen not
+clear what constitues a "major" release verses a "minor" release, the release
+manager can make that call.
+
 After release, inevitably there are bug fixes. It is the responsibility of
 the commiter who fixes a bug to make sure this fix is on the release branch,
 so that we can then create a tagged bug-fix release, which will hopefully
@@ -78,6 +84,8 @@
 
 * Maybe bump the SOABI number in module/imp/importing. This has many
   implications, so make sure the PyPy community agrees to the change.
+  Wheels will use the major.minor release numbers in the name, so bump
+  them if there is an incompatible change to cpyext.
 
 * Update and write documentation
 
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -25,3 +25,12 @@
 
 Test and reduce the probability of a deadlock when acquiring a semaphore by
 moving global state changes closer to the actual aquire.
+
+.. branch: shadowstack-issue2722
+
+Make the shadowstack size more dynamic
+
+.. branch: cffi-libs
+
+Move _ssl and _hashlib from rpython to a cffi-based module, like on python3.
+Reduces the number of problematic linked-in libraries (libssl, libcrypto)
diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -2,14 +2,9 @@
 What's new in PyPy3 7.1+
 ========================
 
-.. this is the revision after release-pypy3.6-v7.1
-.. startrev: d642a3c217cb
+.. this is the revision after release-pypy3.6-v7.1.1
+.. startrev: db5a1e7fbbd0
 
-.. branch: zlib-make-py3-go-boom
+.. branch: fix-literal-prev_digit-underscore
 
-Complain if you try to copy a flushed zlib decompress on py3
-
-.. branch: winoverlapped
-
-Add support for async (overlapped) IO on Windows.
-
+Fix parsing for converting strings with underscore into ints
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -126,6 +126,9 @@
         for c in expressions.constants:
             yield (self.simple_test, "x="+c, "x", eval(c))
 
+    def test_const_underscore(self):
+        yield (self.simple_test, "x=0xffff_ffff_ff20_0000", "x", 0xffffffffff200000)
+
     def test_neg_sys_maxint(self):
         import sys
         stmt = "x = %s" % (-sys.maxint-1)
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -384,6 +384,16 @@
 PyAPI_FUNC(int) PyObject_CallFinalizerFromDealloc(PyObject *);
 #endif
 
+/*
+ * On CPython with Py_REF_DEBUG these use _PyRefTotal, _Py_NegativeRefcount,
+ * _Py_GetRefTotal, ...
+ * So far we ignore Py_REF_DEBUG
+ */
+
+#define _Py_INC_REFTOTAL
+#define _Py_DEC_REFTOTAL
+#define _Py_REF_DEBUG_COMMA
+#define _Py_CHECK_REFCNT(OP)    /* a semicolon */;
 
 /* PyPy internal ----------------------------------- */
 PyAPI_FUNC(int) PyPyType_Register(PyTypeObject *);
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -1462,7 +1462,7 @@
         with open(fname, "w") as f:
             f.write("this is a rename test")
         str_name = str(self.pdir) + '/test_rename.txt'
-        os.rename(self.path, str_name)
+        os.rename(fname, str_name)
         with open(str_name) as f:
             assert f.read() == 'this is a rename test'
         os.rename(str_name, fname)
@@ -1471,6 +1471,11 @@
         with open(unicode_name) as f:
             assert f.read() == 'this is a rename test'
         os.rename(unicode_name, fname)
+        
+        os.rename(bytes(fname, 'utf-8'), bytes(str_name, 'utf-8'))
+        with open(str_name) as f:
+            assert f.read() == 'this is a rename test'
+        os.rename(str_name, fname)
         with open(fname) as f:
             assert f.read() == 'this is a rename test'
         os.unlink(fname)
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -55,9 +55,15 @@
 reserves 768KB of stack space, which should suffice (on Linux,
 depending on the compiler settings) for ~1400 calls.  Setting the
 value to N reserves N/1000 times 768KB of stack space.
+
+Note that there are other factors that also limit the stack size.
+The operating system typically sets a maximum which can be changed
+manually (e.g. with "ulimit" on Linux) for the main thread.  For other
+threads you can configure the limit by calling "threading.stack_size()".
 """
     from rpython.rlib.rstack import _stack_set_length_fraction
     from rpython.rlib.rstackovf import StackOverflow
+    from rpython.rlib.rgc import increase_root_stack_depth
     if new_limit <= 0:
         raise oefmt(space.w_ValueError, "recursion limit must be positive")
     try:
@@ -69,6 +75,7 @@
         raise oefmt(space.w_RecursionError,
                     "maximum recursion depth exceeded")
     space.sys.recursionlimit = new_limit
+    increase_root_stack_depth(int(new_limit * 0.001 * 163840))
 
 def getrecursionlimit(space):
     """Return the last value set by setrecursionlimit().
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -57,7 +57,9 @@
             w_object._check_released(space)
             return W_MemoryView.copy(w_object)
         view = space.buffer_w(w_object, space.BUF_FULL_RO)
-        return view.wrap(space)
+        mv = view.wrap(space)
+        mv.obj = w_object
+        return mv
 
     def _make_descr__cmp(name):
         def descr__cmp(self, space, w_other):
@@ -265,6 +267,10 @@
         # I've never seen anyone filling this field
         return space.newtuple([])
 
+    def w_get_obj(self, space):
+        self._check_released(space)
+        return self.obj
+
     def descr_repr(self, space):
         if self.view is None:
             return self.getrepr(space, 'released memory')
@@ -529,6 +535,7 @@
     shape       = GetSetProperty(W_MemoryView.w_get_shape),
     strides     = GetSetProperty(W_MemoryView.w_get_strides),
     suboffsets  = GetSetProperty(W_MemoryView.w_get_suboffsets),
+    obj         = GetSetProperty(W_MemoryView.w_get_obj),
     _pypy_raw_address = interp2app(W_MemoryView.descr_pypy_raw_address),
     )
 W_MemoryView.typedef.acceptable_as_base_class = False
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -62,13 +62,15 @@
         assert w.tobytes() == bytes(w) == b'geb'
 
     def test_memoryview_attrs(self):
-        v = memoryview(b"a"*100)
+        b = b"a"*100
+        v = memoryview(b)
         assert v.format == "B"
         assert v.itemsize == 1
         assert v.shape == (100,)
         assert v.ndim == 1
         assert v.strides == (1,)
         assert v.nbytes == 100
+        assert v.obj is b
 
     def test_suboffsets(self):
         v = memoryview(b"a"*100)
diff --git a/pypy/tool/build_cffi_imports.py b/pypy/tool/build_cffi_imports.py
--- a/pypy/tool/build_cffi_imports.py
+++ b/pypy/tool/build_cffi_imports.py
@@ -14,7 +14,7 @@
     "tk": "_tkinter/tklib_build.py",
     "curses": "_curses_build.py" if sys.platform != "win32" else None,
     "syslog": "_syslog_build.py" if sys.platform != "win32" else None,
-    "_gdbm": "_gdbm_build.py"  if sys.platform != "win32" else None,
+    "gdbm": "_gdbm_build.py"  if sys.platform != "win32" else None,
     "pwdgrp": "_pwdgrp_build.py" if sys.platform != "win32" else None,
     "resource": "_resource_build.py" if sys.platform != "win32" else None,
     "lzma": "_lzma_build.py",
diff --git a/pypy/tool/pytest/test/test_appsupport.py b/pypy/tool/pytest/test/test_appsupport.py
--- a/pypy/tool/pytest/test/test_appsupport.py
+++ b/pypy/tool/pytest/test/test_appsupport.py
@@ -37,25 +37,6 @@
 
 class TestSpaceConfig:
     @pytest.mark.xfail(reason="Can't check config with -A in pypy3")
-    def test_applevel_skipped_on_cpython_and_spaceconfig(self, testdir):
-        setpypyconftest(testdir)
-        testdir.makepyfile("""
-            class AppTestClass:
-                spaceconfig = {"objspace.usemodules._random": True}
-                def setup_class(cls):
-                    assert 0
-                def test_applevel(self):
-                    pass
-        """)
-        result = testdir.runpytest("-A")
-        assert result.ret == 0
-        if hasattr(sys, 'pypy_translation_info') and \
-           sys.pypy_translation_info.get('objspace.usemodules._random'):
-            result.stdout.fnmatch_lines(["*1 error*"])
-        else:
-            # setup_class didn't get called, otherwise it would error
-            result.stdout.fnmatch_lines(["*1 skipped*"])
-
     def test_interp_spaceconfig(self, testdir):
         setpypyconftest(testdir)
         p = testdir.makepyfile("""
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -572,6 +572,8 @@
             self.move_out_of_nursery_ptr = getfn(GCClass.move_out_of_nursery,
                                               [s_gc, SomeAddress()],
                                               SomeAddress())
+        if hasattr(self.root_walker, 'build_increase_root_stack_depth_ptr'):
+            self.root_walker.build_increase_root_stack_depth_ptr(getfn)
 
 
     def create_custom_trace_funcs(self, gc, rtyper):
@@ -1652,6 +1654,12 @@
         else:
             hop.rename("same_as")
 
+    def gct_gc_increase_root_stack_depth(self, hop):
+        if not hasattr(self.root_walker, 'gc_increase_root_stack_depth_ptr'):
+            return
+        hop.genop("direct_call",
+                  [self.root_walker.gc_increase_root_stack_depth_ptr,
+                   hop.spaceop.args[0]])
 
 
 class TransformerLayoutBuilder(gctypelayout.TypeLayoutBuilder):
diff --git a/rpython/memory/gctransform/shadowstack.py b/rpython/memory/gctransform/shadowstack.py
--- a/rpython/memory/gctransform/shadowstack.py
+++ b/rpython/memory/gctransform/shadowstack.py
@@ -140,6 +140,7 @@
         # this is a dict {tid: SHADOWSTACKREF}, where the tid for the
         # current thread may be missing so far
         gcdata.thread_stacks = None
+        shadow_stack_pool.has_threads = True
 
         # Return the thread identifier, as an integer.
         get_tid = rthread.get_ident
@@ -252,6 +253,15 @@
         self.gc_modified_shadowstack_ptr = getfn(gc_modified_shadowstack,
                                                  [], annmodel.s_None)
 
+    def build_increase_root_stack_depth_ptr(self, getfn):
+        shadow_stack_pool = self.shadow_stack_pool
+        def gc_increase_root_stack_depth(new_size):
+            shadow_stack_pool.increase_root_stack_depth(new_size)
+
+        self.gc_increase_root_stack_depth_ptr = getfn(
+                gc_increase_root_stack_depth, [annmodel.SomeInteger()],
+                annmodel.s_None)
+
     def postprocess_graph(self, gct, graph, any_inlining):
         from rpython.memory.gctransform import shadowcolor
         if any_inlining:
@@ -269,6 +279,7 @@
     """
     _alloc_flavor_ = "raw"
     root_stack_depth = 163840
+    has_threads = False
 
     def __init__(self, gcdata):
         self.unused_full_stack = llmemory.NULL
@@ -337,6 +348,44 @@
             if self.unused_full_stack == llmemory.NULL:
                 raise MemoryError
 
+    def increase_root_stack_depth(self, new_depth):
+        if new_depth <= self.root_stack_depth:
+            return     # can't easily decrease the size
+        if self.unused_full_stack:
+            llmemory.raw_free(self.unused_full_stack)
+            self.unused_full_stack = llmemory.NULL
+        used = self.gcdata.root_stack_top - self.gcdata.root_stack_base
+        addr = self._resize(self.gcdata.root_stack_base, used, new_depth)
+        self.gcdata.root_stack_base = addr
+        self.gcdata.root_stack_top  = addr + used
+        # no gc operations above: we just switched shadowstacks
+        if self.has_threads:
+            self._resize_thread_shadowstacks(new_depth)
+        self.root_stack_depth = new_depth
+
+    def _resize_thread_shadowstacks(self, new_depth):
+        if self.gcdata.thread_stacks is not None:
+            for ssref in self.gcdata.thread_stacks.values():
+                if ssref.base:
+                    used = ssref.top - ssref.base
+                    addr = self._resize(ssref.base, used, new_depth)
+                    ssref.base = addr
+                    ssref.top = addr + used
+    _resize_thread_shadowstacks._dont_inline_ = True
+
+    def _resize(self, base, used, new_depth):
+        new_size = sizeofaddr * new_depth
+        ll_assert(used <= new_size, "shadowstack resize: overflow detected")
+        addr = llmemory.raw_malloc(new_size)
+        if addr == llmemory.NULL:
+            raise MemoryError
+        # note that we don't know the total memory size of 'base', but we
+        # know the size of the part that is used right now, and we only need
+        # to copy that
+        llmemory.raw_memmove(base, addr, used)
+        llmemory.raw_free(base)
+        return addr
+
 
 def get_shadowstackref(root_walker, gctransformer):
     if hasattr(gctransformer, '_SHADOWSTACKREF'):
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -690,6 +690,13 @@
         hop.exception_cannot_occur()
         return hop.genop('gc_move_out_of_nursery', hop.args_v, resulttype=hop.r_result)
 
+ at jit.dont_look_inside
+def increase_root_stack_depth(new_depth):
+    """Shadowstack: make sure the size of the shadowstack is at least
+    'new_depth' pointers."""
+    from rpython.rtyper.lltypesystem.lloperation import llop
+    llop.gc_increase_root_stack_depth(lltype.Void, new_depth)
+
 # ____________________________________________________________
 
 
diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py
--- a/rpython/rlib/rstring.py
+++ b/rpython/rlib/rstring.py
@@ -579,6 +579,11 @@
         assert i >= 0
         self.i = i
         c = self.s[i]
+        if self.allow_underscores and c == '_':
+            i = self.i - 1
+            assert i >= 0
+            self.i = i
+            c = self.s[i]
         digit = ord(c)
         if '0' <= c <= '9':
             digit -= ord('0')
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -1226,6 +1226,9 @@
     def op_gc_move_out_of_nursery(self, obj):
         raise NotImplementedError("gc_move_out_of_nursery")
 
+    def op_gc_increase_root_stack_depth(self, new_depth):
+        raise NotImplementedError("gc_increase_root_stack_depth")
+
     def op_revdb_stop_point(self, *args):
         pass
     def op_revdb_send_answer(self, *args):
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -530,6 +530,7 @@
     'gc_rawrefcount_next_dead':         LLOp(),
 
     'gc_move_out_of_nursery':           LLOp(),
+    'gc_increase_root_stack_depth':     LLOp(canrun=True),
 
     'gc_push_roots'        : LLOp(),  # temporary: list of roots to save
     'gc_pop_roots'         : LLOp(),  # temporary: list of roots to restore
diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py
--- a/rpython/rtyper/lltypesystem/opimpl.py
+++ b/rpython/rtyper/lltypesystem/opimpl.py
@@ -776,6 +776,9 @@
 def op_gc_move_out_of_nursery(obj):
     return obj
 
+def op_gc_increase_root_stack_depth(new_depth):
+    pass
+
 def op_revdb_do_next_call():
     pass
 
diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py
--- a/rpython/translator/c/gc.py
+++ b/rpython/translator/c/gc.py
@@ -450,9 +450,8 @@
         # XXX hard-code the field name here
         gcpol_ss = '%s->gcd_inst_root_stack_top' % funcgen.expr(c_gcdata)
         #
-        yield ('typedef struct { void %s; } pypy_ss_t;'
+        yield ('typedef struct { char %s; } pypy_ss_t;'
                    % ', '.join(['*s%d' % i for i in range(numcolors)]))
-        yield 'pypy_ss_t *ss;'
         funcgen.gcpol_ss = gcpol_ss
 
     def OP_GC_PUSH_ROOTS(self, funcgen, op):
@@ -462,26 +461,29 @@
         raise Exception("gc_pop_roots should be removed by postprocess_graph")
 
     def OP_GC_ENTER_ROOTS_FRAME(self, funcgen, op):
-        return 'ss = (pypy_ss_t *)%s; %s = (void *)(ss+1);' % (
-            funcgen.gcpol_ss, funcgen.gcpol_ss)
+        # avoid arithmatic on void*
+        return '({0}) = (char*)({0}) + sizeof(pypy_ss_t);'.format(funcgen.gcpol_ss,)
 
     def OP_GC_LEAVE_ROOTS_FRAME(self, funcgen, op):
-        return '%s = (void *)ss;' % funcgen.gcpol_ss
+        # avoid arithmatic on void*
+        return '({0}) = (char*)({0}) - sizeof(pypy_ss_t);'.format(funcgen.gcpol_ss,)
 
     def OP_GC_SAVE_ROOT(self, funcgen, op):
         num = op.args[0].value
         exprvalue = funcgen.expr(op.args[1])
-        return 'ss->s%d = (void *)%s;\t/* gc_save_root */' % (num, exprvalue)
+        return '((pypy_ss_t *)%s)[-1].s%d = (char *)%s;' % (
+            funcgen.gcpol_ss, num, exprvalue)
 
     def OP_GC_RESTORE_ROOT(self, funcgen, op):
         num = op.args[0].value
         exprvalue = funcgen.expr(op.args[1])
         typename = funcgen.db.gettype(op.args[1].concretetype)
-        result = '%s = (%s)ss->s%d;' % (exprvalue, cdecl(typename, ''), num)
+        result = '%s = (%s)((pypy_ss_t *)%s)[-1].s%d;' % (
+            exprvalue, cdecl(typename, ''), funcgen.gcpol_ss, num)
         if isinstance(op.args[1], Constant):
-            return '/* %s\t* gc_restore_root */' % result
+            return '/* %s */' % result
         else:
-            return '%s\t/* gc_restore_root */' % result
+            return result
 
 
 class AsmGcRootFrameworkGcPolicy(BasicFrameworkGcPolicy):
diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -1912,6 +1912,45 @@
     def test_total_gc_time(self):
         res = self.run("total_gc_time")
         assert res > 0 # should take a few microseconds
+
+    def define_increase_root_stack_depth(cls):
+        class X:
+            pass
+        def g(n):
+            if n <= 0:
+                return None
+            x = X()
+            x.n = n
+            x.next = g(n - 1)
+            return x
+        def f(depth):
+            from rpython.rlib.rstack import _stack_set_length_fraction
+            _stack_set_length_fraction(50.0)
+            # ^^^ the default is enough for at least 10'000 (but less than
+            # 100'000) recursions of the simple function g().  We multiply
+            # it by 50.0 to make sure that 200'000 works.  The default
+            # shadowstack depth is 163'840 entries, so 200'000 overflows
+            # that default shadowstack depth, and gives a segfault unless
+            # the following line works too.
+            from rpython.rlib.rgc import increase_root_stack_depth
+            increase_root_stack_depth(depth + 100)
+            #
+            g(depth)
+            return 42
+        return f
+
+    def test_increase_root_stack_depth(self):
+        if not sys.platform.startswith('linux'):
+            py.test.skip("linux only")
+        #
+        def myrunner(args):
+            args1 = ['/bin/bash', '-c', 'ulimit -s unlimited && %s' %
+                     (' '.join(args),)]
+            return subprocess.check_output(args1)
+        res = self.run("increase_root_stack_depth", 200000, runner=myrunner)
+        assert res == 42
+
+
 # ____________________________________________________________________
 
 class TaggedPointersTest(object):
diff --git a/rpython/translator/platform/arch/s390x.py b/rpython/translator/platform/arch/s390x.py
old mode 100644
new mode 100755
--- a/rpython/translator/platform/arch/s390x.py
+++ b/rpython/translator/platform/arch/s390x.py
@@ -88,6 +88,8 @@
         return "zEC12"
     if machine == 0x2964:
         return "z13"
+    if machine == 0x3907:  # gcc supports z14 as of 2019/05/08
+        return "z14"
 
     # well all others are unsupported!
     return "unknown"

From pypy.commits at gmail.com  Mon May 27 00:43:16 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 26 May 2019 21:43:16 -0700 (PDT)
Subject: [pypy-commit] pypy default: fixes for win32
Message-ID: <5ceb6ae4.1c69fb81.5752e.dcba@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r96691:6dd0f9f9ab4e
Date: 2019-05-27 07:29 +0300
http://bitbucket.org/pypy/pypy/changeset/6dd0f9f9ab4e/

Log:	fixes for win32

diff --git a/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py b/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py
--- a/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py
@@ -1,101 +1,101 @@
 from _pypy_openssl import lib, ffi
-
-
-def enum_certificates(store_name):
-    """Retrieve certificates from Windows' cert store.
-
-store_name may be one of 'CA', 'ROOT' or 'MY'.  The system may provide
-more cert storages, too.  The function returns a list of (bytes,
-encoding_type, trust) tuples.  The encoding_type flag can be interpreted
-with X509_ASN_ENCODING or PKCS_7_ASN_ENCODING. The trust setting is either
-a set of OIDs or the boolean True.
-    """
-    hStore = lib.CertOpenStore(lib.CERT_STORE_PROV_SYSTEM_A, 0, ffi.NULL,
-                               lib.CERT_STORE_READONLY_FLAG | lib.CERT_SYSTEM_STORE_LOCAL_MACHINE,
-                               bytes(store_name, "ascii"))
-    if hStore == ffi.NULL:
-        raise WindowsError(*ffi.getwinerror())
-    
-    result = []
-    pCertCtx = ffi.NULL
-    try:
-        while True:
-            pCertCtx = lib.CertEnumCertificatesInStore(hStore, pCertCtx)
-            if pCertCtx == ffi.NULL:
-                break
-            cert = ffi.buffer(pCertCtx.pbCertEncoded, pCertCtx.cbCertEncoded)[:]
-            enc = certEncodingType(pCertCtx.dwCertEncodingType)
-            keyusage = parseKeyUsage(pCertCtx, lib.CERT_FIND_PROP_ONLY_ENHKEY_USAGE_FLAG)
-            if keyusage is True:
-                keyusage = parseKeyUsage(pCertCtx, lib.CERT_FIND_EXT_ONLY_ENHKEY_USAGE_FLAG)
-            result.append((cert, enc, keyusage))
-    finally:
-        if pCertCtx != ffi.NULL:
-            lib.CertFreeCertificateContext(pCertCtx)
-        if not lib.CertCloseStore(hStore, 0):
-            # This error case might shadow another exception.
-            raise WindowsError(*ffi.getwinerror())
-    return result
-
-
-def enum_crls(store_name):
-    """Retrieve CRLs from Windows' cert store.
-
-store_name may be one of 'CA', 'ROOT' or 'MY'.  The system may provide
-more cert storages, too.  The function returns a list of (bytes,
-encoding_type) tuples.  The encoding_type flag can be interpreted with
-X509_ASN_ENCODING or PKCS_7_ASN_ENCODING."""
-    hStore = lib.CertOpenStore(lib.CERT_STORE_PROV_SYSTEM_A, 0, ffi.NULL,
-                               lib.CERT_STORE_READONLY_FLAG | lib.CERT_SYSTEM_STORE_LOCAL_MACHINE,
-                               bytes(store_name, "ascii"))
-    if hStore == ffi.NULL:
-        raise WindowsError(*ffi.getwinerror())
-
-    result = []
-    pCrlCtx = ffi.NULL
-    try:
-        while True:
-            pCrlCtx = lib.CertEnumCRLsInStore(hStore, pCrlCtx)
-            if pCrlCtx == ffi.NULL:
-                break
-            crl = ffi.buffer(pCrlCtx.pbCrlEncoded, pCrlCtx.cbCrlEncoded)[:]
-            enc = certEncodingType(pCrlCtx.dwCertEncodingType)
-            result.append((crl, enc))
-    finally:
-        if pCrlCtx != ffi.NULL:
-            lib.CertFreeCRLContext(pCrlCtx)
-        if not lib.CertCloseStore(hStore, 0):
-            # This error case might shadow another exception.
-            raise WindowsError(*ffi.getwinerror())
-    return result
-
-
-def certEncodingType(encodingType):
-    if encodingType == lib.X509_ASN_ENCODING:
-        return "x509_asn"
-    if encodingType == lib.PKCS_7_ASN_ENCODING:
-        return "pkcs_7_asn"
-    return encodingType
-
-def parseKeyUsage(pCertCtx, flags):
-    pSize = ffi.new("DWORD *")
-    if not lib.CertGetEnhancedKeyUsage(pCertCtx, flags, ffi.NULL, pSize):
-        error_with_message = ffi.getwinerror()
-        if error_with_message[0] == lib.CRYPT_E_NOT_FOUND:
-            return True
-        raise WindowsError(*error_with_message)
-
-    pUsageMem = ffi.new("char[]", pSize[0])
-    pUsage = ffi.cast("PCERT_ENHKEY_USAGE", pUsageMem)
-    if not lib.CertGetEnhancedKeyUsage(pCertCtx, flags, pUsage, pSize):
-        error_with_message = ffi.getwinerror()
-        if error_with_message[0] == lib.CRYPT_E_NOT_FOUND:
-            return True
-        raise WindowsError(*error_with_message)
-
-    retval = set()
-    for i in range(pUsage.cUsageIdentifier):
-        if pUsage.rgpszUsageIdentifier[i]:
-            oid = ffi.string(pUsage.rgpszUsageIdentifier[i]).decode('ascii')
-            retval.add(oid)
-    return retval
+
+
+def enum_certificates(store_name):
+    """Retrieve certificates from Windows' cert store.
+
+store_name may be one of 'CA', 'ROOT' or 'MY'.  The system may provide
+more cert storages, too.  The function returns a list of (bytes,
+encoding_type, trust) tuples.  The encoding_type flag can be interpreted
+with X509_ASN_ENCODING or PKCS_7_ASN_ENCODING. The trust setting is either
+a set of OIDs or the boolean True.
+    """
+    hStore = lib.CertOpenStore(lib.CERT_STORE_PROV_SYSTEM_A, 0, ffi.NULL,
+                               lib.CERT_STORE_READONLY_FLAG | lib.CERT_SYSTEM_STORE_LOCAL_MACHINE,
+                               bytes(store_name))
+    if hStore == ffi.NULL:
+        raise WindowsError(*ffi.getwinerror())
+
+    result = []
+    pCertCtx = ffi.NULL
+    try:
+        while True:
+            pCertCtx = lib.CertEnumCertificatesInStore(hStore, pCertCtx)
+            if pCertCtx == ffi.NULL:
+                break
+            cert = ffi.buffer(pCertCtx.pbCertEncoded, pCertCtx.cbCertEncoded)[:]
+            enc = certEncodingType(pCertCtx.dwCertEncodingType)
+            keyusage = parseKeyUsage(pCertCtx, lib.CERT_FIND_PROP_ONLY_ENHKEY_USAGE_FLAG)
+            if keyusage is True:
+                keyusage = parseKeyUsage(pCertCtx, lib.CERT_FIND_EXT_ONLY_ENHKEY_USAGE_FLAG)
+            result.append((cert, enc, keyusage))
+    finally:
+        if pCertCtx != ffi.NULL:
+            lib.CertFreeCertificateContext(pCertCtx)
+        if not lib.CertCloseStore(hStore, 0):
+            # This error case might shadow another exception.
+            raise WindowsError(*ffi.getwinerror())
+    return result
+
+
+def enum_crls(store_name):
+    """Retrieve CRLs from Windows' cert store.
+
+store_name may be one of 'CA', 'ROOT' or 'MY'.  The system may provide
+more cert storages, too.  The function returns a list of (bytes,
+encoding_type) tuples.  The encoding_type flag can be interpreted with
+X509_ASN_ENCODING or PKCS_7_ASN_ENCODING."""
+    hStore = lib.CertOpenStore(lib.CERT_STORE_PROV_SYSTEM_A, 0, ffi.NULL,
+                               lib.CERT_STORE_READONLY_FLAG | lib.CERT_SYSTEM_STORE_LOCAL_MACHINE,
+                               bytes(store_name))
+    if hStore == ffi.NULL:
+        raise WindowsError(*ffi.getwinerror())
+
+    result = []
+    pCrlCtx = ffi.NULL
+    try:
+        while True:
+            pCrlCtx = lib.CertEnumCRLsInStore(hStore, pCrlCtx)
+            if pCrlCtx == ffi.NULL:
+                break
+            crl = ffi.buffer(pCrlCtx.pbCrlEncoded, pCrlCtx.cbCrlEncoded)[:]
+            enc = certEncodingType(pCrlCtx.dwCertEncodingType)
+            result.append((crl, enc))
+    finally:
+        if pCrlCtx != ffi.NULL:
+            lib.CertFreeCRLContext(pCrlCtx)
+        if not lib.CertCloseStore(hStore, 0):
+            # This error case might shadow another exception.
+            raise WindowsError(*ffi.getwinerror())
+    return result
+
+
+def certEncodingType(encodingType):
+    if encodingType == lib.X509_ASN_ENCODING:
+        return "x509_asn"
+    if encodingType == lib.PKCS_7_ASN_ENCODING:
+        return "pkcs_7_asn"
+    return encodingType
+
+def parseKeyUsage(pCertCtx, flags):
+    pSize = ffi.new("DWORD *")
+    if not lib.CertGetEnhancedKeyUsage(pCertCtx, flags, ffi.NULL, pSize):
+        error_with_message = ffi.getwinerror()
+        if error_with_message[0] == lib.CRYPT_E_NOT_FOUND:
+            return True
+        raise WindowsError(*error_with_message)
+
+    pUsageMem = ffi.new("char[]", pSize[0])
+    pUsage = ffi.cast("PCERT_ENHKEY_USAGE", pUsageMem)
+    if not lib.CertGetEnhancedKeyUsage(pCertCtx, flags, pUsage, pSize):
+        error_with_message = ffi.getwinerror()
+        if error_with_message[0] == lib.CRYPT_E_NOT_FOUND:
+            return True
+        raise WindowsError(*error_with_message)
+
+    retval = set()
+    for i in range(pUsage.cUsageIdentifier):
+        if pUsage.rgpszUsageIdentifier[i]:
+            oid = ffi.string(pUsage.rgpszUsageIdentifier[i]).decode('ascii')
+            retval.add(oid)
+    return retval
diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py
--- a/rpython/rlib/_rsocket_rffi.py
+++ b/rpython/rlib/_rsocket_rffi.py
@@ -225,6 +225,7 @@
                         ('SOL_UDP', 17),
                         ('SOMAXCONN', 5),
                         ('IPPROTO_IP', 6),
+                        ('IPPROTO_IPV6', 41),
                         ('IPPROTO_ICMP', 1),
                         ('IPPROTO_TCP', 6),
                         ('IPPROTO_UDP', 17),

From pypy.commits at gmail.com  Mon May 27 00:43:18 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 26 May 2019 21:43:18 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: merge default into py3.6
Message-ID: <5ceb6ae6.1c69fb81.e48e7.a427@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r96692:6912df235ecc
Date: 2019-05-27 07:36 +0300
http://bitbucket.org/pypy/pypy/changeset/6912df235ecc/

Log:	merge default into py3.6

diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
@@ -40,6 +40,11 @@
  */
 static const long Cryptography_HAS_OP_NO_COMPRESSION;
 
+/* Internally invented symbol to tell us if SSL_OP_ENABLE_MIDDLEBOX_COMPAT is
+ * supported
+ */
+static const long Cryptography_HAS_OP_ENABLE_MIDDLEBOX_COMPAT;
+
 static const long Cryptography_HAS_SSL_OP_MSIE_SSLV2_RSA_PADDING;
 static const long Cryptography_HAS_SSL_SET_SSL_CTX;
 static const long Cryptography_HAS_SSL_OP_NO_TICKET;
@@ -73,6 +78,7 @@
 static const long SSL_OP_NETSCAPE_REUSE_CIPHER_CHANGE_BUG;
 static const long SSL_OP_NO_SSLv2;
 static const long SSL_OP_SSLREF2_REUSE_CERT_TYPE_BUG;
+static const long SSL_OP_ENABLE_MIDDLEBOX_COMPAT;
 static const long SSL_OP_MICROSOFT_BIG_SSLV3_BUFFER;
 static const long SSL_OP_MSIE_SSLV2_RSA_PADDING;
 static const long SSL_OP_SSLEAY_080_CLIENT_DH_BUG;
@@ -560,6 +566,13 @@
 const long SSL_OP_NO_COMPRESSION = 0;
 #endif
 
+#ifdef SSL_OP_ENABLE_MIDDLEBOX_COMPAT
+static const long Cryptography_HAS_OP_ENABLE_MIDDLEBOX_COMPAT = 1;
+#else
+static const long Cryptography_HAS_OP_ENABLE_MIDDLEBOX_COMPAT = 0;
+const long SSL_OP_ENABLE_MIDDLEBOX_COMPAT = 0;
+#endif
+
 #ifdef SSL_OP_NO_TLSv1_1
 static const long Cryptography_HAS_TLSv1_1 = 1;
 #else
diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -20,6 +20,7 @@
         pyerr_write_unraisable)
 from _cffi_ssl._stdssl import error
 from select import select
+import socket
 from enum import IntEnum as _IntEnum
 
 if sys.platform == 'win32':
@@ -221,6 +222,7 @@
     def _new__ssl_socket(sslctx, sock, socket_type, server_hostname, inbio, outbio):
         self = _SSLSocket(sslctx)
         ctx = sslctx.ctx
+        self.owner = ssl_sock  # weakref
 
         if server_hostname:
             self.server_hostname = server_hostname.decode('idna', 'strict')
@@ -289,7 +291,8 @@
     def owner(self, value):
         if value is None:
             self._owner = None
-        self._owner = weakref.ref(value)
+        else:
+            self._owner = weakref.ref(value)
 
     @property
     def context(self):
@@ -311,9 +314,6 @@
         return self.socket_type == SSL_SERVER
 
     def do_handshake(self):
-        # delay to prevent circular imports
-        import socket
-
         sock = self.get_socket_or_connection_gone()
         ssl = self.ssl
         timeout = _socket_timeout(sock)
@@ -385,9 +385,6 @@
                 return _decode_certificate(self.peer_cert)
 
     def write(self, bytestring):
-        # delay to prevent circular imports
-        import socket
-
         deadline = 0
         b = _str_to_ffi_buffer(bytestring)
         sock = self.get_socket_or_connection_gone()
@@ -446,9 +443,6 @@
             raise pyssl_error(self, length)
 
     def read(self, length, buffer_into=None):
-        # delay to prevent circular imports
-        import socket
-
         ssl = self.ssl
 
         if length < 0 and buffer_into is None:
@@ -589,9 +583,6 @@
         return sock
 
     def shutdown(self):
-        # delay to prevent circular imports
-        import socket
-
         sock = self.get_socket_or_None()
         nonblocking = False
         ssl = self.ssl
@@ -1068,7 +1059,7 @@
             self._add_ca_certs(buf, len(buf), ca_file_type)
 
         # load cafile or capath
-        if cafile or capath:
+        if cafile is not None or capath is not None:
             if cafile is None:
                 cafilebuf = ffi.NULL
             else:
@@ -1577,3 +1568,5 @@
                            "enough data to seed the PRNG");
         return bytecount
 
+socket.RAND_add = RAND_add
+socket.RAND_status = RAND_status
diff --git a/lib_pypy/_cffi_ssl/_stdssl/error.py b/lib_pypy/_cffi_ssl/_stdssl/error.py
--- a/lib_pypy/_cffi_ssl/_stdssl/error.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/error.py
@@ -26,6 +26,13 @@
         if self.strerror and isinstance(self.strerror, str):
             return self.strerror
         return str(self.args)
+# these are expected on socket as well
+socket.sslerror = SSLError
+for v in [ 'SSL_ERROR_ZERO_RETURN', 'SSL_ERROR_WANT_READ',
+     'SSL_ERROR_WANT_WRITE', 'SSL_ERROR_WANT_X509_LOOKUP', 'SSL_ERROR_SYSCALL',
+     'SSL_ERROR_SSL', 'SSL_ERROR_WANT_CONNECT', 'SSL_ERROR_EOF',
+     'SSL_ERROR_INVALID_ERROR_CODE' ]:
+    setattr(socket, v, locals()[v]) 
 
 class SSLZeroReturnError(SSLError):
     """ SSL/TLS session closed cleanly. """
diff --git a/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py b/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py
--- a/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py
@@ -1,101 +1,101 @@
 from _pypy_openssl import lib, ffi
-
-
-def enum_certificates(store_name):
-    """Retrieve certificates from Windows' cert store.
-
-store_name may be one of 'CA', 'ROOT' or 'MY'.  The system may provide
-more cert storages, too.  The function returns a list of (bytes,
-encoding_type, trust) tuples.  The encoding_type flag can be interpreted
-with X509_ASN_ENCODING or PKCS_7_ASN_ENCODING. The trust setting is either
-a set of OIDs or the boolean True.
-    """
-    hStore = lib.CertOpenStore(lib.CERT_STORE_PROV_SYSTEM_A, 0, ffi.NULL,
-                               lib.CERT_STORE_READONLY_FLAG | lib.CERT_SYSTEM_STORE_LOCAL_MACHINE,
-                               bytes(store_name, "ascii"))
-    if hStore == ffi.NULL:
-        raise WindowsError(*ffi.getwinerror())
-    
-    result = []
-    pCertCtx = ffi.NULL
-    try:
-        while True:
-            pCertCtx = lib.CertEnumCertificatesInStore(hStore, pCertCtx)
-            if pCertCtx == ffi.NULL:
-                break
-            cert = ffi.buffer(pCertCtx.pbCertEncoded, pCertCtx.cbCertEncoded)[:]
-            enc = certEncodingType(pCertCtx.dwCertEncodingType)
-            keyusage = parseKeyUsage(pCertCtx, lib.CERT_FIND_PROP_ONLY_ENHKEY_USAGE_FLAG)
-            if keyusage is True:
-                keyusage = parseKeyUsage(pCertCtx, lib.CERT_FIND_EXT_ONLY_ENHKEY_USAGE_FLAG)
-            result.append((cert, enc, keyusage))
-    finally:
-        if pCertCtx != ffi.NULL:
-            lib.CertFreeCertificateContext(pCertCtx)
-        if not lib.CertCloseStore(hStore, 0):
-            # This error case might shadow another exception.
-            raise WindowsError(*ffi.getwinerror())
-    return result
-
-
-def enum_crls(store_name):
-    """Retrieve CRLs from Windows' cert store.
-
-store_name may be one of 'CA', 'ROOT' or 'MY'.  The system may provide
-more cert storages, too.  The function returns a list of (bytes,
-encoding_type) tuples.  The encoding_type flag can be interpreted with
-X509_ASN_ENCODING or PKCS_7_ASN_ENCODING."""
-    hStore = lib.CertOpenStore(lib.CERT_STORE_PROV_SYSTEM_A, 0, ffi.NULL,
-                               lib.CERT_STORE_READONLY_FLAG | lib.CERT_SYSTEM_STORE_LOCAL_MACHINE,
-                               bytes(store_name, "ascii"))
-    if hStore == ffi.NULL:
-        raise WindowsError(*ffi.getwinerror())
-
-    result = []
-    pCrlCtx = ffi.NULL
-    try:
-        while True:
-            pCrlCtx = lib.CertEnumCRLsInStore(hStore, pCrlCtx)
-            if pCrlCtx == ffi.NULL:
-                break
-            crl = ffi.buffer(pCrlCtx.pbCrlEncoded, pCrlCtx.cbCrlEncoded)[:]
-            enc = certEncodingType(pCrlCtx.dwCertEncodingType)
-            result.append((crl, enc))
-    finally:
-        if pCrlCtx != ffi.NULL:
-            lib.CertFreeCRLContext(pCrlCtx)
-        if not lib.CertCloseStore(hStore, 0):
-            # This error case might shadow another exception.
-            raise WindowsError(*ffi.getwinerror())
-    return result
-
-
-def certEncodingType(encodingType):
-    if encodingType == lib.X509_ASN_ENCODING:
-        return "x509_asn"
-    if encodingType == lib.PKCS_7_ASN_ENCODING:
-        return "pkcs_7_asn"
-    return encodingType
-
-def parseKeyUsage(pCertCtx, flags):
-    pSize = ffi.new("DWORD *")
-    if not lib.CertGetEnhancedKeyUsage(pCertCtx, flags, ffi.NULL, pSize):
-        error_with_message = ffi.getwinerror()
-        if error_with_message[0] == lib.CRYPT_E_NOT_FOUND:
-            return True
-        raise WindowsError(*error_with_message)
-
-    pUsageMem = ffi.new("char[]", pSize[0])
-    pUsage = ffi.cast("PCERT_ENHKEY_USAGE", pUsageMem)
-    if not lib.CertGetEnhancedKeyUsage(pCertCtx, flags, pUsage, pSize):
-        error_with_message = ffi.getwinerror()
-        if error_with_message[0] == lib.CRYPT_E_NOT_FOUND:
-            return True
-        raise WindowsError(*error_with_message)
-
-    retval = set()
-    for i in range(pUsage.cUsageIdentifier):
-        if pUsage.rgpszUsageIdentifier[i]:
-            oid = ffi.string(pUsage.rgpszUsageIdentifier[i]).decode('ascii')
-            retval.add(oid)
-    return retval
+
+
+def enum_certificates(store_name):
+    """Retrieve certificates from Windows' cert store.
+
+store_name may be one of 'CA', 'ROOT' or 'MY'.  The system may provide
+more cert storages, too.  The function returns a list of (bytes,
+encoding_type, trust) tuples.  The encoding_type flag can be interpreted
+with X509_ASN_ENCODING or PKCS_7_ASN_ENCODING. The trust setting is either
+a set of OIDs or the boolean True.
+    """
+    hStore = lib.CertOpenStore(lib.CERT_STORE_PROV_SYSTEM_A, 0, ffi.NULL,
+                               lib.CERT_STORE_READONLY_FLAG | lib.CERT_SYSTEM_STORE_LOCAL_MACHINE,
+                               bytes(store_name))
+    if hStore == ffi.NULL:
+        raise WindowsError(*ffi.getwinerror())
+
+    result = []
+    pCertCtx = ffi.NULL
+    try:
+        while True:
+            pCertCtx = lib.CertEnumCertificatesInStore(hStore, pCertCtx)
+            if pCertCtx == ffi.NULL:
+                break
+            cert = ffi.buffer(pCertCtx.pbCertEncoded, pCertCtx.cbCertEncoded)[:]
+            enc = certEncodingType(pCertCtx.dwCertEncodingType)
+            keyusage = parseKeyUsage(pCertCtx, lib.CERT_FIND_PROP_ONLY_ENHKEY_USAGE_FLAG)
+            if keyusage is True:
+                keyusage = parseKeyUsage(pCertCtx, lib.CERT_FIND_EXT_ONLY_ENHKEY_USAGE_FLAG)
+            result.append((cert, enc, keyusage))
+    finally:
+        if pCertCtx != ffi.NULL:
+            lib.CertFreeCertificateContext(pCertCtx)
+        if not lib.CertCloseStore(hStore, 0):
+            # This error case might shadow another exception.
+            raise WindowsError(*ffi.getwinerror())
+    return result
+
+
+def enum_crls(store_name):
+    """Retrieve CRLs from Windows' cert store.
+
+store_name may be one of 'CA', 'ROOT' or 'MY'.  The system may provide
+more cert storages, too.  The function returns a list of (bytes,
+encoding_type) tuples.  The encoding_type flag can be interpreted with
+X509_ASN_ENCODING or PKCS_7_ASN_ENCODING."""
+    hStore = lib.CertOpenStore(lib.CERT_STORE_PROV_SYSTEM_A, 0, ffi.NULL,
+                               lib.CERT_STORE_READONLY_FLAG | lib.CERT_SYSTEM_STORE_LOCAL_MACHINE,
+                               bytes(store_name))
+    if hStore == ffi.NULL:
+        raise WindowsError(*ffi.getwinerror())
+
+    result = []
+    pCrlCtx = ffi.NULL
+    try:
+        while True:
+            pCrlCtx = lib.CertEnumCRLsInStore(hStore, pCrlCtx)
+            if pCrlCtx == ffi.NULL:
+                break
+            crl = ffi.buffer(pCrlCtx.pbCrlEncoded, pCrlCtx.cbCrlEncoded)[:]
+            enc = certEncodingType(pCrlCtx.dwCertEncodingType)
+            result.append((crl, enc))
+    finally:
+        if pCrlCtx != ffi.NULL:
+            lib.CertFreeCRLContext(pCrlCtx)
+        if not lib.CertCloseStore(hStore, 0):
+            # This error case might shadow another exception.
+            raise WindowsError(*ffi.getwinerror())
+    return result
+
+
+def certEncodingType(encodingType):
+    if encodingType == lib.X509_ASN_ENCODING:
+        return "x509_asn"
+    if encodingType == lib.PKCS_7_ASN_ENCODING:
+        return "pkcs_7_asn"
+    return encodingType
+
+def parseKeyUsage(pCertCtx, flags):
+    pSize = ffi.new("DWORD *")
+    if not lib.CertGetEnhancedKeyUsage(pCertCtx, flags, ffi.NULL, pSize):
+        error_with_message = ffi.getwinerror()
+        if error_with_message[0] == lib.CRYPT_E_NOT_FOUND:
+            return True
+        raise WindowsError(*error_with_message)
+
+    pUsageMem = ffi.new("char[]", pSize[0])
+    pUsage = ffi.cast("PCERT_ENHKEY_USAGE", pUsageMem)
+    if not lib.CertGetEnhancedKeyUsage(pCertCtx, flags, pUsage, pSize):
+        error_with_message = ffi.getwinerror()
+        if error_with_message[0] == lib.CRYPT_E_NOT_FOUND:
+            return True
+        raise WindowsError(*error_with_message)
+
+    retval = set()
+    for i in range(pUsage.cUsageIdentifier):
+        if pUsage.rgpszUsageIdentifier[i]:
+            oid = ffi.string(pUsage.rgpszUsageIdentifier[i]).decode('ascii')
+            retval.add(oid)
+    return retval
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -34,3 +34,7 @@
 
 Move _ssl and _hashlib from rpython to a cffi-based module, like on python3.
 Reduces the number of problematic linked-in libraries (libssl, libcrypto)
+
+.. branch: fix-vmprof-memory-tracking
+
+Fix a bug that prevent memory-tracking in vmprof working on PyPy.
diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -60,11 +60,6 @@
     'interval' is a float representing the sampling interval, in seconds.
     Must be smaller than 1.0
     """
-    w_modules = space.sys.get('modules')
-    #if space.contains_w(w_modules, space.newtext('_continuation')):
-    #    space.warn(space.newtext("Using _continuation/greenlet/stacklet together "
-    #                             "with vmprof will crash"),
-    #               space.w_RuntimeWarning)
     try:
         rvmprof.enable(fileno, period, memory, native, real_time)
     except rvmprof.VMProfError as e:
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -3989,55 +3989,6 @@
         # here it works again
         self.check_operations_history(guard_class=0, record_exact_class=1)
 
-    def test_record_exact_class_nonconst(self):
-        class Base(object):
-            def f(self):
-                raise NotImplementedError
-            def g(self):
-                raise NotImplementedError
-        class A(Base):
-            def f(self):
-                return self.a
-            def g(self):
-                return self.a + 1
-        class B(Base):
-            def f(self):
-                return self.b
-            def g(self):
-                return self.b + 1
-        class C(B):
-            def f(self):
-                self.c += 1
-                return self.c
-            def g(self):
-                return self.c + 1
-        @dont_look_inside
-        def make(x):
-            if x > 0:
-                a = A()
-                a.a = x + 1
-            elif x < 0:
-                a = B()
-                a.b = -x
-            else:
-                a = C()
-                a.c = 10
-            return a, type(a)
-        def f(x):
-            a, cls = make(x)
-            record_exact_class(a, cls)
-            if x > 0:
-                z = a.f()
-            elif x < 0:
-                z = a.f()
-            else:
-                z = a.f()
-            return z + a.g()
-        res1 = f(6)
-        res2 = self.interp_operations(f, [6])
-        assert res1 == res2
-        self.check_operations_history(guard_class=1, record_exact_class=0)
-
     def test_generator(self):
         def g(n):
             yield n+1
@@ -4889,3 +4840,52 @@
 
         res = self.meta_interp(f, [0])
         assert res == f(0)
+
+    def test_record_exact_class_nonconst(self):
+        class Base(object):
+            def f(self):
+                raise NotImplementedError
+            def g(self):
+                raise NotImplementedError
+        class A(Base):
+            def f(self):
+                return self.a
+            def g(self):
+                return self.a + 1
+        class B(Base):
+            def f(self):
+                return self.b
+            def g(self):
+                return self.b + 1
+        class C(B):
+            def f(self):
+                self.c += 1
+                return self.c
+            def g(self):
+                return self.c + 1
+        @dont_look_inside
+        def make(x):
+            if x > 0:
+                a = A()
+                a.a = x + 1
+            elif x < 0:
+                a = B()
+                a.b = -x
+            else:
+                a = C()
+                a.c = 10
+            return a, type(a)
+        def f(x):
+            a, cls = make(x)
+            record_exact_class(a, cls)
+            if x > 0:
+                z = a.f()
+            elif x < 0:
+                z = a.f()
+            else:
+                z = a.f()
+            return z + a.g()
+        res1 = f(6)
+        res2 = self.interp_operations(f, [6])
+        assert res1 == res2
+        self.check_operations_history(guard_class=1, record_exact_class=0)
diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py
--- a/rpython/rlib/_rsocket_rffi.py
+++ b/rpython/rlib/_rsocket_rffi.py
@@ -225,6 +225,7 @@
                         ('SOL_UDP', 17),
                         ('SOMAXCONN', 5),
                         ('IPPROTO_IP', 6),
+                        ('IPPROTO_IPV6', 41),
                         ('IPPROTO_ICMP', 1),
                         ('IPPROTO_TCP', 6),
                         ('IPPROTO_UDP', 17),
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -143,7 +143,7 @@
             native = 0 # force disabled on Windows
         lines = 0 # not supported on PyPy currently
 
-        p_error = self.cintf.vmprof_init(fileno, interval, lines, memory, "pypy", native, real_time)
+        p_error = self.cintf.vmprof_init(fileno, interval, memory, lines, "pypy", native, real_time)
         if p_error:
             raise VMProfError(rffi.charp2str(p_error))
 
diff --git a/rpython/rlib/rvmprof/test/test_rvmprof.py b/rpython/rlib/rvmprof/test/test_rvmprof.py
--- a/rpython/rlib/rvmprof/test/test_rvmprof.py
+++ b/rpython/rlib/rvmprof/test/test_rvmprof.py
@@ -98,12 +98,12 @@
         self.tmpfilename = str(self.tmpfile)
         super(RVMProfSamplingTest, self).init()
 
-    ENTRY_POINT_ARGS = (int, float)
-    def entry_point(self, value, delta_t):
+    ENTRY_POINT_ARGS = (int, float, int)
+    def entry_point(self, value, delta_t, memory=0):
         code = self.MyCode('py:code:52:test_enable')
         rvmprof.register_code(code, self.MyCode.get_name)
         fd = os.open(self.tmpfilename, os.O_WRONLY | os.O_CREAT, 0666)
-        rvmprof.enable(fd, self.SAMPLING_INTERVAL)
+        rvmprof.enable(fd, self.SAMPLING_INTERVAL, memory=memory)
         start = time.time()
         res = 0
         while time.time() < start+delta_t:
@@ -128,17 +128,25 @@
 
     def test(self):
         from vmprof import read_profile
-        assert self.entry_point(10**4, 0.1) == 99990000
+        assert self.entry_point(10**4, 0.1, 0) == 99990000
         assert self.tmpfile.check()
         self.tmpfile.remove()
         #
-        assert self.rpy_entry_point(10**4, 0.5) == 99990000
+        assert self.rpy_entry_point(10**4, 0.5, 0) == 99990000
         assert self.tmpfile.check()
         prof = read_profile(self.tmpfilename)
         tree = prof.get_tree()
         assert tree.name == 'py:code:52:test_enable'
         assert self.approx_equal(tree.count, 0.5/self.SAMPLING_INTERVAL)
 
+    def test_mem(self):
+        from vmprof import read_profile
+        assert self.rpy_entry_point(10**4, 0.5, 1) == 99990000
+        assert self.tmpfile.check()
+        prof = read_profile(self.tmpfilename)
+        assert prof.profile_memory
+        assert all(p[-1] > 0 for p in prof.profiles)
+
 
 class TestNative(RVMProfSamplingTest):
 
@@ -177,7 +185,7 @@
     def test(self):
         from vmprof import read_profile
         # from vmprof.show import PrettyPrinter
-        assert self.rpy_entry_point(3, 0.5) == 42000
+        assert self.rpy_entry_point(3, 0.5, 0) == 42000
         assert self.tmpfile.check()
 
         prof = read_profile(self.tmpfilename)

From pypy.commits at gmail.com  Mon May 27 00:43:21 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 26 May 2019 21:43:21 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: fix merge
Message-ID: <5ceb6ae9.1c69fb81.b9cb0.6309@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r96693:f91bbed46ef7
Date: 2019-05-27 07:37 +0300
http://bitbucket.org/pypy/pypy/changeset/f91bbed46ef7/

Log:	fix merge

diff --git a/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py b/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py
--- a/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py
@@ -12,10 +12,10 @@
     """
     hStore = lib.CertOpenStore(lib.CERT_STORE_PROV_SYSTEM_A, 0, ffi.NULL,
                                lib.CERT_STORE_READONLY_FLAG | lib.CERT_SYSTEM_STORE_LOCAL_MACHINE,
-                               bytes(store_name))
+                               bytes(store_name, "ascii"))
     if hStore == ffi.NULL:
         raise WindowsError(*ffi.getwinerror())
-
+    
     result = []
     pCertCtx = ffi.NULL
     try:
@@ -47,7 +47,7 @@
 X509_ASN_ENCODING or PKCS_7_ASN_ENCODING."""
     hStore = lib.CertOpenStore(lib.CERT_STORE_PROV_SYSTEM_A, 0, ffi.NULL,
                                lib.CERT_STORE_READONLY_FLAG | lib.CERT_SYSTEM_STORE_LOCAL_MACHINE,
-                               bytes(store_name))
+                               bytes(store_name, "ascii"))
     if hStore == ffi.NULL:
         raise WindowsError(*ffi.getwinerror())
 

From pypy.commits at gmail.com  Mon May 27 00:43:23 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 26 May 2019 21:43:23 -0700 (PDT)
Subject: [pypy-commit] pypy winmultiprocessing: close branch to be merged
Message-ID: <5ceb6aeb.1c69fb81.10301.1201@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: winmultiprocessing
Changeset: r96694:f46824e4d92c
Date: 2019-05-27 07:40 +0300
http://bitbucket.org/pypy/pypy/changeset/f46824e4d92c/

Log:	close branch to be merged

diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -8,3 +8,7 @@
 .. branch: fix-literal-prev_digit-underscore
 
 Fix parsing for converting strings with underscore into ints
+
+.. branch: winmultiprocessing
+
+Improve multiprocessing support on win32

From pypy.commits at gmail.com  Mon May 27 00:43:25 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 26 May 2019 21:43:25 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: merge winmultiprocessing which starts
 support for multiprocessing on win32
Message-ID: <5ceb6aed.1c69fb81.f2653.0cbb@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r96695:8bab4372bd45
Date: 2019-05-27 07:41 +0300
http://bitbucket.org/pypy/pypy/changeset/8bab4372bd45/

Log:	merge winmultiprocessing which starts support for multiprocessing on
	win32

diff --git a/lib_pypy/_overlapped.py b/lib_pypy/_overlapped.py
--- a/lib_pypy/_overlapped.py
+++ b/lib_pypy/_overlapped.py
@@ -173,7 +173,6 @@
         olderr = _kernel32.GetLastError()
         hascompletedio = HasOverlappedIoCompleted(self.overlapped[0])      
         if not hascompletedio and self.type != OverlappedType.TYPE_NOT_STARTED:
-            
             wait = _kernel32.CancelIoEx(self.handle, self.overlapped)
             ret = self.GetOverlappedResult(wait)
             err = _winapi.ERROR_SUCCESS
@@ -186,6 +185,7 @@
                SetFromWindowsErr(err)
         if self.overlapped[0].hEvent != 0:
             _winapi.CloseHandle(self.overlapped[0].hEvent)
+        _winapi.SetLastError(olderr)
 
     @property
     def event(self):
@@ -216,10 +216,6 @@
         else:
             return transferred[0]
 
-    def getbuffer(self):
-        xxx
-        return None
-
     def cancel(self):
         result = True
         if self.type == OverlappedType.TYPE_NOT_STARTED or self.type == OverlappedType.TYPE_WAIT_NAMED_PIPE_AND_CONNECT:
diff --git a/lib_pypy/_pypy_testcapi.py b/lib_pypy/_pypy_testcapi.py
--- a/lib_pypy/_pypy_testcapi.py
+++ b/lib_pypy/_pypy_testcapi.py
@@ -64,7 +64,6 @@
 
     compiler = new_compiler()
     compiler.output_dir = output_dir
-
     # Compile .c file
     include_dir = os.path.join(thisdir, '..', 'include')
     if sys.platform == 'win32':
@@ -80,7 +79,7 @@
     output_filename = modulename + _get_c_extension_suffix()
     if sys.platform == 'win32':
         libname = 'python{0[0]}{0[1]}'.format(sys.version_info)
-        library = os.path.join(thisdir, '..', 'lib', libname)
+        library = os.path.join(thisdir, '..', 'libs', libname)
         if not os.path.exists(library + '.lib'):
             # For a local translation or nightly build
             library = os.path.join(thisdir, '..', 'pypy', 'goal', libname)
diff --git a/lib_pypy/_pypy_winbase_build.py b/lib_pypy/_pypy_winbase_build.py
--- a/lib_pypy/_pypy_winbase_build.py
+++ b/lib_pypy/_pypy_winbase_build.py
@@ -101,8 +101,11 @@
 HANDLE WINAPI CreateFileW(LPCWSTR, DWORD, DWORD, LPSECURITY_ATTRIBUTES,
                    DWORD, DWORD, HANDLE);
 BOOL ReadFile(HANDLE, LPVOID, DWORD, LPDWORD, LPOVERLAPPED);
+BOOL WaitNamedPipeA(LPCSTR, DWORD);
+BOOL WINAPI WriteFile(HANDLE, LPCVOID, DWORD, LPDWORD, LPOVERLAPPED);
 BOOL WINAPI SetNamedPipeHandleState(HANDLE, LPDWORD, LPDWORD, LPDWORD);
 BOOL WINAPI ConnectNamedPipe(HANDLE, LPOVERLAPPED);
+BOOL WINAPI PeekNamedPipe(HANDLE, LPVOID, DWORD, LPDWORD, LPDWORD, LPDWORD);
 HANDLE WINAPI CreateEventA(LPSECURITY_ATTRIBUTES, BOOL, BOOL, LPCSTR);
 HANDLE WINAPI CreateEventW(LPSECURITY_ATTRIBUTES, BOOL, BOOL, LPCWSTR);
 BOOL WINAPI SetEvent(HANDLE);
@@ -110,8 +113,11 @@
 BOOL WINAPI CancelIoEx(HANDLE, LPOVERLAPPED);
 BOOL WINAPI CloseHandle(HANDLE);
 DWORD WINAPI GetLastError(VOID);
+void WINAPI SetLastError(DWORD);
 BOOL WINAPI GetOverlappedResult(HANDLE, LPOVERLAPPED, LPDWORD, BOOL);
 HANDLE WINAPI GetCurrentProcess(void);
+HANDLE OpenProcess(DWORD, BOOL, DWORD);
+void ExitProcess(UINT);
 BOOL WINAPI DuplicateHandle(HANDLE, HANDLE, HANDLE, LPHANDLE,
                             DWORD, BOOL, DWORD);
 BOOL WINAPI CreateProcessA(char *, char *, void *,
@@ -147,7 +153,7 @@
 BOOL WINAPI GetQueuedCompletionStatus(HANDLE, LPDWORD, ULONG**, LPOVERLAPPED*, DWORD);
 HANDLE WINAPI CreateIoCompletionPort(HANDLE, HANDLE, ULONG_PTR, DWORD);
 
-BOOL WINAPI WriteFile(HANDLE, LPCVOID, DWORD, LPDWORD, LPOVERLAPPED);
+
 
 #define WT_EXECUTEINWAITTHREAD 0x00000004
 #define WT_EXECUTEONLYONCE 0x00000008
diff --git a/lib_pypy/_pypy_winbase_cffi.py b/lib_pypy/_pypy_winbase_cffi.py
--- a/lib_pypy/_pypy_winbase_cffi.py
+++ b/lib_pypy/_pypy_winbase_cffi.py
@@ -3,8 +3,8 @@
 
 ffi = _cffi_backend.FFI('_pypy_winbase_cffi',
     _version = 0x2601,
-    _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x68\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x01\x42\x03\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x1A\x03\x00\x01\x3B\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x4C\x03\x00\x00\x27\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x27\x11\x00\x00\x27\x11\x00\x01\x47\x03\x00\x01\x3C\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x03\x00\x00\x33\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x33\x11\x00\x00\x11\x11\x00\x01\x32\x03\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x22\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x22\x11\x00\x00\x21\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x22\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x21\x11\x00\x00\x22\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x01\x48\x03\x00\x00\x0A\x01\x00\x00\x21\x11\x00\x00\x21\x11\x00\x00\x22\x11\x00\x01\x2D\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x5E\x11\x00\x00\x0A\x01\x00\x00\x21\x11\x00\x00\x0A\x01\x00\x00\x22\x11\x00\x00\x63\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x21\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x21\x11\x00\x00\x21\x03\x00\x00\x22\x03\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x21\x11\x00\x00\x21\x11\x00\x00\x21\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x22\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x21\x11\x00\x00\x22\x11\x00\x00\x63\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x21\x11\x00\x00\x22\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x33\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x01\x68\x03\x00\x00\x0A\x01\x00\x00\x21\x11\x00\x00\x22\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\xE2\x03\x00\x00\x07\x01\x00\x01\x4B\x03\x00\x00\x15\x11\x00\x00\x01\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\xB3\x11\x00\x00\xB3\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\xB3\x11\x00\x00\xB3\x11\x00\x00\x2F\x11\x00\x00\x30\x11\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x70\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x1A\x0D\x00\x00\x0A\x01\x00\x00\x33\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x1A\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x1A\x0D\x00\x00\x11\x11\x00\x00\xB3\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x1A\x0D\x00\x00\x02\x0F\x00\x00\xDD\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\xDD\x0D\x00\x00\x00\x0F\x00\x00\xDD\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x11\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x11\x0D\x00\x00\x00\x0F\x00\x00\x11\x0D\x00\x01\x41\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x01\x4C\x03\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xEC\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xE2\x03\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xEF\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xEC\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xEF\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xEC\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xB3\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xEC\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xF5\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xEC\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x01\x68\x0D\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x00\x0F\x00\x01\x68\x0D\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x0C\x09\x00\x01\x38\x03\x00\x00\x13\x09\x00\x01\x3A\x03\x00\x00\x14\x09\x00\x00\x0D\x09\x00\x00\x09\x09\x00\x01\x3E\x03\x00\x00\x0E\x09\x00\x01\x40\x03\x00\x00\x0A\x09\x00\x00\x0F\x09\x00\x00\x15\x09\x00\x01\x46\x03\x00\x01\x45\x03\x00\x00\x17\x09\x00\x00\x16\x09\x00\x00\x0B\x09\x00\x00\x10\x09\x00\x01\x4A\x03\x00\x00\x11\x09\x00\x00\x12\x09\x00\x00\x02\x01\x00\x01\x4C\x05\x00\x00\x00\x0E\x00\x01\x4C\x05\x00\x00\x00\x08\x00\x00\x4D\x03\x00\x00\x53\x03\x00\x00\x98\x03\x00\x00\x05\x01\x00\x00\x01\x09\x00\x00\x04\x09\x00\x00\x07\x09\x00\x00\x08\x09\x00\x00\x00\x09\x00\x00\x02\x09\x00\x00\x03\x09\x00\x00\x05\x09\x00\x00\x06\x09\x00\x01\x5F\x03\x00\x00\x04\x01\x00\x01\x5F\x05\x00\x00\x00\x10\x00\x01\x5F\x05\x00\x00\x00\x08\x00\x00\x1A\x05\x00\x00\x00\x07\x00\x00\xDD\x05\x00\x00\x00\x08\x00\x00\x00\x01\x00\x00\xE2\x05\x00\x00\x01\x00',
-    _globals = (b'\x00\x00\x40\x23CancelIo',0,b'\x00\x00\x43\x23CancelIoEx',0,b'\x00\x00\x40\x23CloseHandle',0,b'\x00\x00\x43\x23ConnectNamedPipe',0,b'\x00\x00\xEB\x23CreateEventA',0,b'\x00\x00\xF1\x23CreateEventW',0,b'\x00\x00\xF7\x23CreateFileA',0,b'\x00\x01\x24\x23CreateFileW',0,b'\x00\x01\x12\x23CreateIoCompletionPort',0,b'\x00\x01\x00\x23CreateNamedPipeA',0,b'\x00\x01\x1A\x23CreateNamedPipeW',0,b'\x00\x00\x32\x23CreatePipe',0,b'\x00\x00\x26\x23CreateProcessA',0,b'\x00\x00\xB9\x23CreateProcessW',0,b'\x00\x00\xA2\x23DuplicateHandle',0,b'\x00\x01\x18\x23GetCurrentProcess',0,b'\x00\x00\x72\x23GetExitCodeProcess',0,b'\x00\x00\xDA\x23GetLastError',0,b'\x00\x00\xD5\x23GetModuleFileNameW',0,b'\x00\x00\x47\x23GetOverlappedResult',0,b'\x00\x00\xE9\x23GetProcessHeap',0,b'\x00\x00\x76\x23GetQueuedCompletionStatus',0,b'\x00\x01\x0F\x23GetStdHandle',0,b'\x00\x00\xDA\x23GetVersion',0,b'\x00\x00\xE4\x23HeapAlloc',0,b'\x00\x00\x18\x23HeapFree',0,b'\xFF\xFF\xFF\x1FMAX_PROTOCOL_CHAIN',7,b'\x00\x00\x83\x23PostQueuedCompletionStatus',0,b'\x00\x00\x1D\x23ReadFile',0,b'\x00\x00\x38\x23RegisterWaitForSingleObject',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\xC8\x23SetErrorMode',0,b'\x00\x00\x40\x23SetEvent',0,b'\x00\x00\x7D\x23SetNamedPipeHandleState',0,b'\x00\x00\x6E\x23TerminateProcess',0,b'\x00\x00\x40\x23UnregisterWait',0,b'\x00\x00\x94\x23UnregisterWaitEx',0,b'\x00\x00\x89\x23WSAIoctl',0,b'\xFF\xFF\xFF\x1FWSAPROTOCOL_LEN',255,b'\x00\x00\x5C\x23WSARecv',0,b'\x00\x00\x65\x23WSASend',0,b'\x00\x00\xB2\x23WSAStringToAddressW',0,b'\xFF\xFF\xFF\x1FWT_EXECUTEINWAITTHREAD',4,b'\xFF\xFF\xFF\x1FWT_EXECUTEONLYONCE',8,b'\x00\x00\xCB\x23WaitForMultipleObjects',0,b'\x00\x00\xD1\x23WaitForSingleObject',0,b'\x00\x00\xAB\x23WriteFile',0,b'\x00\x00\xC5\x23_get_osfhandle',0,b'\x00\x00\x24\x23_getch',0,b'\x00\x00\x24\x23_getche',0,b'\x00\x00\xDF\x23_getwch',0,b'\x00\x00\xDF\x23_getwche',0,b'\x00\x00\x24\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\xE1\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\xDC\x23_ungetwch',0,b'\x00\x00\x13\x23bind',0,b'\x00\x00\x10\x23closesocket',0,b'\x00\x00\xDC\x23htons',0,b'\x00\x01\x0A\x23socket',0),
-    _struct_unions = ((b'\x00\x00\x01\x59\x00\x00\x00\x03$1',b'\x00\x01\x55\x11DUMMYSTRUCTNAME',b'\x00\x00\x11\x11Pointer'),(b'\x00\x00\x01\x55\x00\x00\x00\x02$2',b'\x00\x00\x1A\x11Offset',b'\x00\x00\x1A\x11OffsetHigh'),(b'\x00\x00\x01\x5A\x00\x00\x00\x03$3',b'\x00\x01\x60\x11Byte',b'\x00\x01\x66\x11Word'),(b'\x00\x00\x01\x5B\x00\x00\x00\x01$4',b'\x00\x01\x56\x11',b'\x00\x00\x1A\x11Value'),(b'\x00\x00\x01\x56\x00\x00\x00\x02$5',b'\x00\x00\x1A\x13\x00\x00\x00\x1CZone',b'\x00\x00\x1A\x13\x00\x00\x00\x04Level'),(b'\x00\x00\x01\x5C\x00\x00\x00\x03$6',b'\x00\x00\x1A\x11sin6_scope_id',b'\x00\x01\x40\x11sin6_scope_struct'),(b'\x00\x00\x01\x5D\x00\x00\x00\x03$7',b'\x00\x01\x57\x11S_un_b',b'\x00\x01\x58\x11S_un_w',b'\x00\x00\x1A\x11S_addr'),(b'\x00\x00\x01\x57\x00\x00\x00\x02$8',b'\x00\x01\x5F\x11s_b1',b'\x00\x01\x5F\x11s_b2',b'\x00\x01\x5F\x11s_b3',b'\x00\x01\x5F\x11s_b4'),(b'\x00\x00\x01\x58\x00\x00\x00\x02$9',b'\x00\x00\xDD\x11s_w1',b'\x00\x00\xDD\x11s_w2'),(b'\x00\x00\x01\x3C\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x11\x11hProcess',b'\x00\x00\x11\x11hThread',b'\x00\x00\x1A\x11dwProcessId',b'\x00\x00\x1A\x11dwThreadId'),(b'\x00\x00\x01\x40\x00\x00\x00\x00$SCOPE_ID',b'\x00\x01\x5B\x11'),(b'\x00\x00\x01\x47\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x1A\x11cb',b'\x00\x00\x27\x11lpReserved',b'\x00\x00\x27\x11lpDesktop',b'\x00\x00\x27\x11lpTitle',b'\x00\x00\x1A\x11dwX',b'\x00\x00\x1A\x11dwY',b'\x00\x00\x1A\x11dwXSize',b'\x00\x00\x1A\x11dwYSize',b'\x00\x00\x1A\x11dwXCountChars',b'\x00\x00\x1A\x11dwYCountChars',b'\x00\x00\x1A\x11dwFillAttribute',b'\x00\x00\x1A\x11dwFlags',b'\x00\x00\xDD\x11wShowWindow',b'\x00\x00\xDD\x11cbReserved2',b'\x00\x01\x5E\x11lpReserved2',b'\x00\x00\x11\x11hStdInput',b'\x00\x00\x11\x11hStdOutput',b'\x00\x00\x11\x11hStdError'),(b'\x00\x00\x01\x36\x00\x00\x00\x02_GUID',b'\x00\x00\x1A\x11Data1',b'\x00\x00\xDD\x11Data2',b'\x00\x00\xDD\x11Data3',b'\x00\x01\x62\x11Data4'),(b'\x00\x00\x01\x3B\x00\x00\x00\x02_OVERLAPPED',b'\x00\x00\x1A\x11Internal',b'\x00\x00\x1A\x11InternalHigh',b'\x00\x01\x59\x11DUMMYUNIONNAME',b'\x00\x00\x11\x11hEvent'),(b'\x00\x00\x01\x3E\x00\x00\x00\x02_PostCallbackData',b'\x00\x00\x11\x11hCompletionPort',b'\x00\x00\x22\x11Overlapped'),(b'\x00\x00\x01\x41\x00\x00\x00\x02_SECURITY_ATTRIBUTES',b'\x00\x00\x1A\x11nLength',b'\x00\x00\x11\x11lpSecurityDescriptor',b'\x00\x00\x01\x11bInheritHandle'),(b'\x00\x00\x01\x48\x00\x00\x00\x02_WSABUF',b'\x00\x00\x1A\x11len',b'\x00\x00\x27\x11buf'),(b'\x00\x00\x01\x4A\x00\x00\x00\x02_WSAPROTOCOLCHAIN',b'\x00\x00\x01\x11ChainLen',b'\x00\x01\x64\x11ChainEntries'),(b'\x00\x00\x01\x4B\x00\x00\x00\x02_WSAPROTOCOL_INFOW',b'\x00\x00\x1A\x11dwServiceFlags1',b'\x00\x00\x1A\x11dwServiceFlags2',b'\x00\x00\x1A\x11dwServiceFlags3',b'\x00\x00\x1A\x11dwServiceFlags4',b'\x00\x00\x1A\x11dwProviderFlags',b'\x00\x01\x36\x11ProviderId',b'\x00\x00\x1A\x11dwCatalogEntryId',b'\x00\x01\x4A\x11ProtocolChain',b'\x00\x00\x01\x11iVersion',b'\x00\x00\x01\x11iAddressFamily',b'\x00\x00\x01\x11iMaxSockAddr',b'\x00\x00\x01\x11iMinSockAddr',b'\x00\x00\x01\x11iSocketType',b'\x00\x00\x01\x11iProtocol',b'\x00\x00\x01\x11iProtocolMaxOffset',b'\x00\x00\x01\x11iNetworkByteOrder',b'\x00\x00\x01\x11iSecurityScheme',b'\x00\x00\x1A\x11dwMessageSize',b'\x00\x00\x1A\x11dwProviderReserved',b'\x00\x01\x69\x11szProtocol'),(b'\x00\x00\x01\x38\x00\x00\x00\x02in6_addr',b'\x00\x01\x5A\x11u'),(b'\x00\x00\x01\x3A\x00\x00\x00\x02in_addr',b'\x00\x01\x5D\x11S_un'),(b'\x00\x00\x01\x42\x00\x00\x00\x02sockaddr',b'\x00\x00\xDD\x11sa_family',b'\x00\x01\x4D\x11sa_data'),(b'\x00\x00\x01\x46\x00\x00\x00\x02sockaddr_in',b'\x00\x01\x54\x11sin_family',b'\x00\x00\xDD\x11sin_port',b'\x00\x01\x3A\x11sin_addr',b'\x00\x01\x4F\x11sin_zero'),(b'\x00\x00\x01\x45\x00\x00\x00\x00sockaddr_in6',b'\x00\x00\xDD\x11sin6_family',b'\x00\x00\xDD\x11sin6_port',b'\x00\x00\x1A\x11sin6_flowinfo',b'\x00\x01\x38\x11sin6_addr',b'\x00\x01\x5C\x11')),
-    _typenames = (b'\x00\x00\x00\xDDADDRESS_FAMILY',b'\x00\x00\x01\x53AcceptExPtr',b'\x00\x00\x01\x52ConnectExPtr',b'\x00\x00\x01\x51DisconnectExPtr',b'\x00\x00\x01\x36GUID',b'\x00\x00\x01\x38IN6_ADDR',b'\x00\x00\x01\x3AINADDR',b'\x00\x00\x01\x51LPFN_DISCONNECTEX',b'\x00\x00\x01\x37LPIN6_ADDR',b'\x00\x00\x00\x22LPOVERLAPPED',b'\x00\x00\x00\x63LPOVERLAPPED_COMPLETION_ROUTINE',b'\x00\x00\x00\x30LPPROCESS_INFORMATION',b'\x00\x00\x01\x3DLPPostCallbackData',b'\x00\x00\x00\xECLPSECURITY_ATTRIBUTES',b'\x00\x00\x00\x15LPSOCKADDR',b'\x00\x00\x01\x43LPSOCKADDR_IN',b'\x00\x00\x01\x44LPSOCKADDR_IN6_LH',b'\x00\x00\x00\x2FLPSTARTUPINFO',b'\x00\x00\x00\x5ELPWSABUF',b'\x00\x00\x01\x49LPWSAPROTOCOLCHAIN',b'\x00\x00\x00\xB5LPWSAPROTOCOL_INFOW',b'\x00\x00\x01\x3BOVERLAPPED',b'\x00\x00\x01\x37PIN6_ADDR',b'\x00\x00\x01\x39PINADDR',b'\x00\x00\x01\x3CPROCESS_INFORMATION',b'\x00\x00\x01\x3FPSCOPE_ID',b'\x00\x00\x00\xECPSECURITY_ATTRIBUTES',b'\x00\x00\x00\x15PSOCKADDR',b'\x00\x00\x01\x43PSOCKADDR_IN',b'\x00\x00\x01\x44PSOCKADDR_IN6_LH',b'\x00\x00\x01\x3EPostCallbackData',b'\x00\x00\x01\x40SCOPE_ID',b'\x00\x00\x01\x41SECURITY_ATTRIBUTES',b'\x00\x00\x01\x42SOCKADDR',b'\x00\x00\x01\x46SOCKADDR_IN',b'\x00\x00\x01\x45SOCKADDR_IN6_LH',b'\x00\x00\x00\x11SOCKET',b'\x00\x00\x01\x47STARTUPINFO',b'\x00\x00\x00\x3BWAITORTIMERCALLBACK',b'\x00\x00\x01\x48WSABUF',b'\x00\x00\x01\x4AWSAPROTOCOLCHAIN',b'\x00\x00\x01\x4BWSAPROTOCOL_INFOW',b'\x00\x00\x00\xDDwint_t'),
+    _types = b'\x00\x00\x05\x0D\x00\x01\x63\x03\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x01\x7F\x03\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x01\x59\x03\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x03\x00\x01\x52\x03\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x01\x63\x03\x00\x00\x2B\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x2B\x11\x00\x00\x2B\x11\x00\x01\x5E\x03\x00\x01\x53\x03\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x03\x00\x00\x37\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x37\x11\x00\x00\x15\x11\x00\x01\x49\x03\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x26\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x26\x11\x00\x00\x25\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x26\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x19\x11\x00\x00\x07\x01\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x26\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x01\x5F\x03\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x25\x11\x00\x00\x26\x11\x00\x01\x41\x03\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x62\x11\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x0A\x01\x00\x00\x26\x11\x00\x00\x67\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x25\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x25\x11\x00\x00\x25\x03\x00\x00\x26\x03\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x25\x11\x00\x00\x25\x11\x00\x00\x25\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x26\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x26\x11\x00\x00\x67\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x25\x11\x00\x00\x25\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x26\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x37\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x01\x7F\x03\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x26\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\xEE\x03\x00\x00\x07\x01\x00\x01\x62\x03\x00\x00\x19\x11\x00\x00\x05\x03\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\xBF\x11\x00\x00\xBF\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\xBF\x11\x00\x00\xBF\x11\x00\x00\x33\x11\x00\x00\x34\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x74\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x02\x0D\x00\x00\x0A\x01\x00\x00\x37\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x02\x0D\x00\x00\x15\x11\x00\x00\xBF\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x02\x0D\x00\x00\x02\x0F\x00\x00\xE9\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\xE9\x0D\x00\x00\x00\x0F\x00\x00\xE9\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x01\x58\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x01\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\xFD\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xEE\x03\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x01\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xFD\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x01\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xFD\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\xBF\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xFD\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x01\x06\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xFD\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x01\x7F\x0D\x00\x00\x08\x01\x00\x00\x00\x0F\x00\x01\x7F\x0D\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x00\x0F\x00\x01\x7F\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x01\x7F\x0D\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x0C\x09\x00\x01\x4F\x03\x00\x00\x13\x09\x00\x01\x51\x03\x00\x00\x14\x09\x00\x00\x0D\x09\x00\x00\x09\x09\x00\x01\x55\x03\x00\x00\x0E\x09\x00\x01\x57\x03\x00\x00\x0A\x09\x00\x00\x0F\x09\x00\x00\x15\x09\x00\x01\x5D\x03\x00\x01\x5C\x03\x00\x00\x17\x09\x00\x00\x16\x09\x00\x00\x0B\x09\x00\x00\x10\x09\x00\x01\x61\x03\x00\x00\x11\x09\x00\x00\x12\x09\x00\x00\x02\x01\x00\x01\x63\x05\x00\x00\x00\x0E\x00\x01\x63\x05\x00\x00\x00\x08\x00\x00\x51\x03\x00\x00\x57\x03\x00\x00\xA4\x03\x00\x00\x05\x01\x00\x00\x01\x09\x00\x00\x04\x09\x00\x00\x07\x09\x00\x00\x08\x09\x00\x00\x00\x09\x00\x00\x02\x09\x00\x00\x03\x09\x00\x00\x05\x09\x00\x00\x06\x09\x00\x01\x76\x03\x00\x00\x04\x01\x00\x01\x76\x05\x00\x00\x00\x10\x00\x01\x76\x05\x00\x00\x00\x08\x00\x00\x02\x05\x00\x00\x00\x07\x00\x00\xE9\x05\x00\x00\x00\x08\x00\x00\x00\x01\x00\x00\xEE\x05\x00\x00\x01\x00',
+    _globals = (b'\x00\x00\x44\x23CancelIo',0,b'\x00\x00\x47\x23CancelIoEx',0,b'\x00\x00\x44\x23CloseHandle',0,b'\x00\x00\x47\x23ConnectNamedPipe',0,b'\x00\x00\xFC\x23CreateEventA',0,b'\x00\x01\x02\x23CreateEventW',0,b'\x00\x01\x08\x23CreateFileA',0,b'\x00\x01\x35\x23CreateFileW',0,b'\x00\x01\x23\x23CreateIoCompletionPort',0,b'\x00\x01\x11\x23CreateNamedPipeA',0,b'\x00\x01\x2B\x23CreateNamedPipeW',0,b'\x00\x00\x36\x23CreatePipe',0,b'\x00\x00\x2A\x23CreateProcessA',0,b'\x00\x00\xC5\x23CreateProcessW',0,b'\x00\x00\xAE\x23DuplicateHandle',0,b'\x00\x01\x3E\x23ExitProcess',0,b'\x00\x01\x29\x23GetCurrentProcess',0,b'\x00\x00\x76\x23GetExitCodeProcess',0,b'\x00\x00\xE6\x23GetLastError',0,b'\x00\x00\xE1\x23GetModuleFileNameW',0,b'\x00\x00\x4B\x23GetOverlappedResult',0,b'\x00\x00\xFA\x23GetProcessHeap',0,b'\x00\x00\x7A\x23GetQueuedCompletionStatus',0,b'\x00\x01\x20\x23GetStdHandle',0,b'\x00\x00\xE6\x23GetVersion',0,b'\x00\x00\xF5\x23HeapAlloc',0,b'\x00\x00\x1C\x23HeapFree',0,b'\xFF\xFF\xFF\x1FMAX_PROTOCOL_CHAIN',7,b'\x00\x00\xF0\x23OpenProcess',0,b'\x00\x00\x9C\x23PeekNamedPipe',0,b'\x00\x00\x87\x23PostQueuedCompletionStatus',0,b'\x00\x00\x21\x23ReadFile',0,b'\x00\x00\x3C\x23RegisterWaitForSingleObject',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\xD4\x23SetErrorMode',0,b'\x00\x00\x44\x23SetEvent',0,b'\x00\x01\x46\x23SetLastError',0,b'\x00\x00\x81\x23SetNamedPipeHandleState',0,b'\x00\x00\x72\x23TerminateProcess',0,b'\x00\x00\x44\x23UnregisterWait',0,b'\x00\x00\x98\x23UnregisterWaitEx',0,b'\x00\x00\x8D\x23WSAIoctl',0,b'\xFF\xFF\xFF\x1FWSAPROTOCOL_LEN',255,b'\x00\x00\x60\x23WSARecv',0,b'\x00\x00\x69\x23WSASend',0,b'\x00\x00\xBE\x23WSAStringToAddressW',0,b'\xFF\xFF\xFF\x1FWT_EXECUTEINWAITTHREAD',4,b'\xFF\xFF\xFF\x1FWT_EXECUTEONLYONCE',8,b'\x00\x00\xD7\x23WaitForMultipleObjects',0,b'\x00\x00\xDD\x23WaitForSingleObject',0,b'\x00\x00\x00\x23WaitNamedPipeA',0,b'\x00\x00\xB7\x23WriteFile',0,b'\x00\x00\xD1\x23_get_osfhandle',0,b'\x00\x00\x28\x23_getch',0,b'\x00\x00\x28\x23_getche',0,b'\x00\x00\xEB\x23_getwch',0,b'\x00\x00\xEB\x23_getwche',0,b'\x00\x00\x28\x23_kbhit',0,b'\x00\x00\x0B\x23_locking',0,b'\x00\x00\x10\x23_open_osfhandle',0,b'\x00\x00\x04\x23_putch',0,b'\x00\x00\xED\x23_putwch',0,b'\x00\x00\x07\x23_setmode',0,b'\x00\x00\x04\x23_ungetch',0,b'\x00\x00\xE8\x23_ungetwch',0,b'\x00\x00\x17\x23bind',0,b'\x00\x00\x14\x23closesocket',0,b'\x00\x00\xE8\x23htons',0,b'\x00\x01\x1B\x23socket',0),
+    _struct_unions = ((b'\x00\x00\x01\x70\x00\x00\x00\x03$1',b'\x00\x01\x6C\x11DUMMYSTRUCTNAME',b'\x00\x00\x15\x11Pointer'),(b'\x00\x00\x01\x6C\x00\x00\x00\x02$2',b'\x00\x00\x02\x11Offset',b'\x00\x00\x02\x11OffsetHigh'),(b'\x00\x00\x01\x71\x00\x00\x00\x03$3',b'\x00\x01\x77\x11Byte',b'\x00\x01\x7D\x11Word'),(b'\x00\x00\x01\x72\x00\x00\x00\x01$4',b'\x00\x01\x6D\x11',b'\x00\x00\x02\x11Value'),(b'\x00\x00\x01\x6D\x00\x00\x00\x02$5',b'\x00\x00\x02\x13\x00\x00\x00\x1CZone',b'\x00\x00\x02\x13\x00\x00\x00\x04Level'),(b'\x00\x00\x01\x73\x00\x00\x00\x03$6',b'\x00\x00\x02\x11sin6_scope_id',b'\x00\x01\x57\x11sin6_scope_struct'),(b'\x00\x00\x01\x74\x00\x00\x00\x03$7',b'\x00\x01\x6E\x11S_un_b',b'\x00\x01\x6F\x11S_un_w',b'\x00\x00\x02\x11S_addr'),(b'\x00\x00\x01\x6E\x00\x00\x00\x02$8',b'\x00\x01\x76\x11s_b1',b'\x00\x01\x76\x11s_b2',b'\x00\x01\x76\x11s_b3',b'\x00\x01\x76\x11s_b4'),(b'\x00\x00\x01\x6F\x00\x00\x00\x02$9',b'\x00\x00\xE9\x11s_w1',b'\x00\x00\xE9\x11s_w2'),(b'\x00\x00\x01\x53\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x02\x11dwProcessId',b'\x00\x00\x02\x11dwThreadId'),(b'\x00\x00\x01\x57\x00\x00\x00\x00$SCOPE_ID',b'\x00\x01\x72\x11'),(b'\x00\x00\x01\x5E\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x02\x11cb',b'\x00\x00\x2B\x11lpReserved',b'\x00\x00\x2B\x11lpDesktop',b'\x00\x00\x2B\x11lpTitle',b'\x00\x00\x02\x11dwX',b'\x00\x00\x02\x11dwY',b'\x00\x00\x02\x11dwXSize',b'\x00\x00\x02\x11dwYSize',b'\x00\x00\x02\x11dwXCountChars',b'\x00\x00\x02\x11dwYCountChars',b'\x00\x00\x02\x11dwFillAttribute',b'\x00\x00\x02\x11dwFlags',b'\x00\x00\xE9\x11wShowWindow',b'\x00\x00\xE9\x11cbReserved2',b'\x00\x01\x75\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError'),(b'\x00\x00\x01\x4D\x00\x00\x00\x02_GUID',b'\x00\x00\x02\x11Data1',b'\x00\x00\xE9\x11Data2',b'\x00\x00\xE9\x11Data3',b'\x00\x01\x79\x11Data4'),(b'\x00\x00\x01\x52\x00\x00\x00\x02_OVERLAPPED',b'\x00\x00\x02\x11Internal',b'\x00\x00\x02\x11InternalHigh',b'\x00\x01\x70\x11DUMMYUNIONNAME',b'\x00\x00\x15\x11hEvent'),(b'\x00\x00\x01\x55\x00\x00\x00\x02_PostCallbackData',b'\x00\x00\x15\x11hCompletionPort',b'\x00\x00\x26\x11Overlapped'),(b'\x00\x00\x01\x58\x00\x00\x00\x02_SECURITY_ATTRIBUTES',b'\x00\x00\x02\x11nLength',b'\x00\x00\x15\x11lpSecurityDescriptor',b'\x00\x00\x05\x11bInheritHandle'),(b'\x00\x00\x01\x5F\x00\x00\x00\x02_WSABUF',b'\x00\x00\x02\x11len',b'\x00\x00\x2B\x11buf'),(b'\x00\x00\x01\x61\x00\x00\x00\x02_WSAPROTOCOLCHAIN',b'\x00\x00\x05\x11ChainLen',b'\x00\x01\x7B\x11ChainEntries'),(b'\x00\x00\x01\x62\x00\x00\x00\x02_WSAPROTOCOL_INFOW',b'\x00\x00\x02\x11dwServiceFlags1',b'\x00\x00\x02\x11dwServiceFlags2',b'\x00\x00\x02\x11dwServiceFlags3',b'\x00\x00\x02\x11dwServiceFlags4',b'\x00\x00\x02\x11dwProviderFlags',b'\x00\x01\x4D\x11ProviderId',b'\x00\x00\x02\x11dwCatalogEntryId',b'\x00\x01\x61\x11ProtocolChain',b'\x00\x00\x05\x11iVersion',b'\x00\x00\x05\x11iAddressFamily',b'\x00\x00\x05\x11iMaxSockAddr',b'\x00\x00\x05\x11iMinSockAddr',b'\x00\x00\x05\x11iSocketType',b'\x00\x00\x05\x11iProtocol',b'\x00\x00\x05\x11iProtocolMaxOffset',b'\x00\x00\x05\x11iNetworkByteOrder',b'\x00\x00\x05\x11iSecurityScheme',b'\x00\x00\x02\x11dwMessageSize',b'\x00\x00\x02\x11dwProviderReserved',b'\x00\x01\x80\x11szProtocol'),(b'\x00\x00\x01\x4F\x00\x00\x00\x02in6_addr',b'\x00\x01\x71\x11u'),(b'\x00\x00\x01\x51\x00\x00\x00\x02in_addr',b'\x00\x01\x74\x11S_un'),(b'\x00\x00\x01\x59\x00\x00\x00\x02sockaddr',b'\x00\x00\xE9\x11sa_family',b'\x00\x01\x64\x11sa_data'),(b'\x00\x00\x01\x5D\x00\x00\x00\x02sockaddr_in',b'\x00\x01\x6B\x11sin_family',b'\x00\x00\xE9\x11sin_port',b'\x00\x01\x51\x11sin_addr',b'\x00\x01\x66\x11sin_zero'),(b'\x00\x00\x01\x5C\x00\x00\x00\x00sockaddr_in6',b'\x00\x00\xE9\x11sin6_family',b'\x00\x00\xE9\x11sin6_port',b'\x00\x00\x02\x11sin6_flowinfo',b'\x00\x01\x4F\x11sin6_addr',b'\x00\x01\x73\x11')),
+    _typenames = (b'\x00\x00\x00\xE9ADDRESS_FAMILY',b'\x00\x00\x01\x6AAcceptExPtr',b'\x00\x00\x01\x69ConnectExPtr',b'\x00\x00\x01\x68DisconnectExPtr',b'\x00\x00\x01\x4DGUID',b'\x00\x00\x01\x4FIN6_ADDR',b'\x00\x00\x01\x51INADDR',b'\x00\x00\x01\x68LPFN_DISCONNECTEX',b'\x00\x00\x01\x4ELPIN6_ADDR',b'\x00\x00\x00\x26LPOVERLAPPED',b'\x00\x00\x00\x67LPOVERLAPPED_COMPLETION_ROUTINE',b'\x00\x00\x00\x34LPPROCESS_INFORMATION',b'\x00\x00\x01\x54LPPostCallbackData',b'\x00\x00\x00\xFDLPSECURITY_ATTRIBUTES',b'\x00\x00\x00\x19LPSOCKADDR',b'\x00\x00\x01\x5ALPSOCKADDR_IN',b'\x00\x00\x01\x5BLPSOCKADDR_IN6_LH',b'\x00\x00\x00\x33LPSTARTUPINFO',b'\x00\x00\x00\x62LPWSABUF',b'\x00\x00\x01\x60LPWSAPROTOCOLCHAIN',b'\x00\x00\x00\xC1LPWSAPROTOCOL_INFOW',b'\x00\x00\x01\x52OVERLAPPED',b'\x00\x00\x01\x4EPIN6_ADDR',b'\x00\x00\x01\x50PINADDR',b'\x00\x00\x01\x53PROCESS_INFORMATION',b'\x00\x00\x01\x56PSCOPE_ID',b'\x00\x00\x00\xFDPSECURITY_ATTRIBUTES',b'\x00\x00\x00\x19PSOCKADDR',b'\x00\x00\x01\x5APSOCKADDR_IN',b'\x00\x00\x01\x5BPSOCKADDR_IN6_LH',b'\x00\x00\x01\x55PostCallbackData',b'\x00\x00\x01\x57SCOPE_ID',b'\x00\x00\x01\x58SECURITY_ATTRIBUTES',b'\x00\x00\x01\x59SOCKADDR',b'\x00\x00\x01\x5DSOCKADDR_IN',b'\x00\x00\x01\x5CSOCKADDR_IN6_LH',b'\x00\x00\x00\x15SOCKET',b'\x00\x00\x01\x5ESTARTUPINFO',b'\x00\x00\x00\x3FWAITORTIMERCALLBACK',b'\x00\x00\x01\x5FWSABUF',b'\x00\x00\x01\x61WSAPROTOCOLCHAIN',b'\x00\x00\x01\x62WSAPROTOCOL_INFOW',b'\x00\x00\x00\xE9wint_t'),
 )
diff --git a/lib_pypy/_winapi.py b/lib_pypy/_winapi.py
--- a/lib_pypy/_winapi.py
+++ b/lib_pypy/_winapi.py
@@ -5,6 +5,7 @@
 """
 
 import sys
+
 if sys.platform != 'win32':
     raise ImportError("The '_winapi' module is only available on Windows")
 
@@ -16,6 +17,12 @@
 GetVersion = _kernel32.GetVersion
 NULL = _ffi.NULL
 
+def SetLastError(errno):
+    return _kernel32.SetLastError(errno)
+
+def GetLastError():
+    return _kernel32.GetLastError()
+
 # Now the _subprocess module implementation
 def _WinError(type=WindowsError):
     code, message = _ffi.getwinerror()
@@ -51,19 +58,19 @@
     res = _kernel32.CreatePipe(handles, handles + 1, NULL, size)
 
     if not res:
-        SetFromWindowsErr(0)
+        SetFromWindowsErr(GetLastError())
 
     return _handle2int(handles[0]), _handle2int(handles[1])
 
 def CreateNamedPipe(*args):
     handle = _kernel32.CreateNamedPipeW(*args)
-    if handle == INVALID_HANDLE_VALUE:
+    if handle == _INVALID_HANDLE_VALUE:
         SetFromWindowsErr(0)
     return _handle2int(handle)
 
 def CreateFile(*args):
     handle = _kernel32.CreateFileW(*args)
-    if handle == INVALID_HANDLE_VALUE:
+    if handle == _INVALID_HANDLE_VALUE:
         SetFromWindowsErr(0)
     return _handle2int(handle)
 
@@ -77,14 +84,14 @@
         d2 = NULL
     else:
         d2 = _ffi.new('DWORD[1]', [collect_data_timeout])
-    ret = _kernel32.SetNamedPipeHandleState(namedpipe, d0, d1, d2)
+    ret = _kernel32.SetNamedPipeHandleState(_int2handle(namedpipe), d0, d1, d2)
     if not ret:
         raise _WinError()
 
 class Overlapped(object):
     def __init__(self, handle):
         self.overlapped = _ffi.new('OVERLAPPED[1]')
-        self.handle = handle
+        self.handle = _handle2int(handle)
         self.readbuffer = None
         self.pending = 0
         self.completed = 0
@@ -96,47 +103,140 @@
         # do this somehow else
         err = _kernel32.GetLastError()
         bytes = _ffi.new('DWORD[1]')
-        o = self.overlapped[0]
         if self.pending:
-            if _kernel32.CancelIoEx(o.handle, o.overlapped) & \
-                self.GetOverlappedResult(o.handle, o.overlapped, _ffi.addressof(bytes), True):
+            result = _kernel32.CancelIoEx(_int2handle(self.handle), self.overlapped)
+            if result: 
+                _kernel32.GetOverlappedResult(_int2handle(self.handle), self.overlapped, bytes, True)
                 # The operation is no longer pending, nothing to do
-                pass
-            else:
-                raise RuntimeError('deleting an overlapped struct with a pending operation not supported')
+                
+            #else:
+                # We need to raise a warning here and not crash pypy
+                #raise RuntimeError('deleting an overlapped struct with a pending operation not supported')
+        CloseHandle(_int2handle(self.overlapped[0].hEvent))
+        _kernel32.SetLastError(err)
 
     @property
     def event(self):
-        xxx
-        return None
+        return _handle2int(self.overlapped[0].hEvent)
 
     def GetOverlappedResult(self, wait):
         transferred = _ffi.new('DWORD[1]', [0])
-        res = _kernel32.GetOverlappedResult(self.handle, self.overlapped, transferred, wait != 0)
+        res = _kernel32.GetOverlappedResult(_int2handle(self.handle), self.overlapped, transferred, wait != 0)
+        
         if res:
             err = ERROR_SUCCESS
         else:
-            err = GetLastError()
+            err = _kernel32.GetLastError()
+
         if err in (ERROR_SUCCESS, ERROR_MORE_DATA, ERROR_OPERATION_ABORTED):
             self.completed = 1
             self.pending = 0
-        elif res == ERROR_IO_INCOMPLETE:
-            pass
-        else:
+        elif res != ERROR_IO_INCOMPLETE:
             self.pending = 0
-            raise _WinError()
+            raise _WinError(IOError)
+
         if self.completed and self.readbuffer:
-            if transferred != len(self.readbuffer):
-                raise _WinError()
+            if transferred[0] != len(self.readbuffer):
+                tempbuffer = _ffi.new("CHAR[]", transferred[0])
+                _ffi.memmove(tempbuffer, self.readbuffer, transferred[0])
+                self.readbuffer = tempbuffer
         return transferred[0], err
 
     def getbuffer(self):
-        xxx
+        if not self.completed:
+            raise ValueError("can't get read buffer before GetOverlappedResult() "
+                        "signals the operation completed")
+        if self.readbuffer:
+            result = _ffi.buffer(self.readbuffer)
+        else:
+            result = None
+        return result
+
+    def cancel(self):
+        ret = True
+        if self.pending:
+            ret = _kernel32.CancelIoEx(_int2handle(self.handle), self.overlapped)
+        if not ret and _kernel32.GetLastError() != ERROR_NOT_FOUND:
+            return _WinError(IOError)
+        self.pending = 0
         return None
 
-    def cancel(self):
-        xxx
-        return None
+
+def ReadFile(handle, size, overlapped):
+    nread = _ffi.new("DWORD*")
+    err = _ffi.new("DWORD*")
+    use_overlapped = overlapped
+    overlapped = None
+
+    buf = _ffi.new("CHAR[]", size)
+    if not buf:
+        return _ffi.NULL
+
+    if use_overlapped:
+        overlapped = Overlapped(handle)
+        if not overlapped:
+            return _ffi.NULL
+        overlapped.readbuffer = buf
+    
+    if overlapped:
+        ret = _kernel32.ReadFile(_int2handle(handle), buf, size, nread,
+                       overlapped.overlapped)
+    else:
+        ret = _kernel32.ReadFile(_int2handle(handle), buf, size, nread,
+                       _ffi.NULL)
+ 
+    if ret:
+        err = 0
+    else: 
+        err = _kernel32.GetLastError()
+
+    if use_overlapped:
+        if not ret:
+            if err == ERROR_IO_PENDING:
+                overlapped.pending = 1
+            elif err != ERROR_MORE_DATA:
+                return _WinError(IOError)
+        return overlapped, err
+
+    if (not ret) and err != ERROR_MORE_DATA:
+        return _WinError(IOError)
+    return buf, err
+
+def WriteFile(handle, buffer, overlapped=False):
+    written = _ffi.new("DWORD*")
+    err = _ffi.new("DWORD*")
+    use_overlapped = overlapped
+    overlapped = None    
+    if use_overlapped:
+        overlapped = Overlapped(handle)
+        if not overlapped:
+            return _ffi.NULL
+        overlapped.writebuffer = bytes(buffer)
+        buf = overlapped.writebuffer
+    else:
+        buf = _ffi.new("CHAR[]", bytes(buffer))
+    if use_overlapped:
+        ret = _kernel32.WriteFile(_int2handle(handle), buf , len(buf), written, overlapped.overlapped)
+    else:
+        ret = _kernel32.WriteFile(_int2handle(handle), buf , len(buf), written, _ffi.NULL)
+
+    if ret:
+        err = 0
+    else: 
+        err = _kernel32.GetLastError()
+
+    if use_overlapped:
+        if not ret:
+            if err == ERROR_IO_PENDING:
+                overlapped.pending = 1
+            else:
+                return _WinError(IOError)
+        return overlapped, err
+
+    if not ret:
+        return _WinError(IOError)
+     
+    return written[0], err
 
  
 def ConnectNamedPipe(handle, overlapped=False):
@@ -176,7 +276,7 @@
 
     if not res:
         raise _WinError()
-
+    
     return _handle2int(target[0])
 
 def _Z(input):
@@ -225,18 +325,65 @@
             pi.dwProcessId,
             pi.dwThreadId)
 
+def OpenProcess(desired_access, inherit_handle, process_id):
+    handle = _kernel32.OpenProcess(desired_access, inherit_handle, process_id)
+    if handle == _ffi.NULL:
+        SetFromWindowsErr(0)
+        handle = INVALID_HANDLE_VALUE
+
+    return _handle2int(handle)
+
+def PeekNamedPipe(handle, size=0):
+    nread = _ffi.new("DWORD*")
+    navail = _ffi.new("DWORD*")
+    nleft = _ffi.new("DWORD*")
+
+    if size < 0:
+        raise ValueError("negative size")
+
+    if size:
+        buf = _ffi.new("CHAR[]", size)
+        if not buf:
+            return _ffi.NULL
+
+        ret = _kernel32.PeekNamedPipe(_int2handle(handle), buf, size, nread,
+                                      navail, nleft)
+        if not ret:
+            # In CPython SetExcFromWindowsErr is called here.
+            # Not sure what that is doing currently.
+            SetFromWindowsErr(0)
+
+
+        return  buf, navail[0], nleft[0]
+    else:
+        ret = _kernel32.PeekNamedPipe(_int2handle(handle), _ffi.NULL, 0, _ffi.NULL, navail, nleft)
+        if not ret:
+            # In CPython SetExcFromWindowsErr is called here.
+            # Not sure what that is doing currently.
+            SetFromWindowsErr(0)
+        return  navail[0], nleft[0]
+
 def WaitForSingleObject(handle, milliseconds):
     # CPython: the first argument is expected to be an integer.
     res = _kernel32.WaitForSingleObject(_int2handle(handle), milliseconds)
     if res < 0:
         raise _WinError()
+    return res
 
-    return res
+
+def WaitNamedPipe(namedpipe, milliseconds):
+    namedpipe = _ffi.new("CHAR[]", namedpipe.encode("ascii", "ignore"))
+    res = _kernel32.WaitNamedPipeA(namedpipe, milliseconds)
+
+    if res < 0:
+        raise SetFromWindowsErr(0)
+
 
 def WaitForMultipleObjects(handle_sequence, waitflag, milliseconds):
     if len(handle_sequence) > MAXIMUM_WAIT_OBJECTS:
         return None
-    
+    handle_sequence = list(map(_int2handle, handle_sequence))
+    handle_sequence = _ffi.new("HANDLE[]", handle_sequence)
     # CPython makes the wait interruptible by ctrl-c. We need to add this in at some point
     res = _kernel32.WaitForMultipleObjects(len(handle_sequence), handle_sequence, waitflag, milliseconds)
 
@@ -288,6 +435,9 @@
         raise _WinError()
     return _ffi.string(buf)
 
+def ExitProcess(exitcode):
+    _kernel32.ExitProcess(exitcode)
+
 ZERO_MEMORY = 0x00000008
 
 def malloc(size):
@@ -301,6 +451,7 @@
 STD_OUTPUT_HANDLE = -11
 STD_ERROR_HANDLE = -12
 DUPLICATE_SAME_ACCESS = 2
+DUPLICATE_CLOSE_SOURCE = 1
 STARTF_USESTDHANDLES = 0x100
 STARTF_USESHOWWINDOW = 0x001
 SW_HIDE = 0
@@ -318,15 +469,19 @@
 ERROR_SUCCESS           = 0
 ERROR_NETNAME_DELETED   = 64
 ERROR_BROKEN_PIPE       = 109
-ERROR_PIPE_BUSY         = 231 
+ERROR_SEM_TIMEOUT       = 121
+ERROR_PIPE_BUSY         = 231
+ERROR_NO_DATA           = 232 
 ERROR_MORE_DATA         = 234
 ERROR_PIPE_CONNECTED    = 535
 ERROR_OPERATION_ABORTED = 995
 ERROR_IO_INCOMPLETE     = 996
 ERROR_IO_PENDING        = 997
+ERROR_NOT_FOUND          = 1168
 ERROR_CONNECTION_REFUSED = 1225
 ERROR_CONNECTION_ABORTED = 1236
 
+
 PIPE_ACCESS_INBOUND = 0x00000001
 PIPE_ACCESS_OUTBOUND = 0x00000002
 PIPE_ACCESS_DUPLEX   = 0x00000003
@@ -362,6 +517,26 @@
 NMPWAIT_NOWAIT                =  0x00000001
 NMPWAIT_USE_DEFAULT_WAIT      =  0x00000000
 
+FILE_READ_DATA = 1
+FILE_WRITE_DATA = 2
+FILE_APPEND_DATA = 4
+FILE_READ_EA = 8
+FILE_WRITE_EA = 16
+FILE_EXECUTE = 32
+FILE_READ_ATTRIBUTES = 128 
+FILE_WRITE_ATTRIBUTES = 256
+READ_CONTROL = 0x00020000
+SYNCHRONIZE = 0x00100000
+STANDARD_RIGHTS_EXECUTE = READ_CONTROL
+STANDARD_RIGHTS_READ = READ_CONTROL
+STANDARD_RIGHTS_WRITE = READ_CONTROL
+
+FILE_GENERIC_EXECUTE = FILE_EXECUTE | FILE_READ_ATTRIBUTES | STANDARD_RIGHTS_EXECUTE | SYNCHRONIZE
+FILE_GENERIC_READ = FILE_READ_ATTRIBUTES | FILE_READ_DATA | FILE_READ_EA | STANDARD_RIGHTS_READ | SYNCHRONIZE
+FILE_GENERIC_WRITE = FILE_APPEND_DATA | FILE_WRITE_ATTRIBUTES | FILE_WRITE_DATA | FILE_WRITE_EA | STANDARD_RIGHTS_WRITE | SYNCHRONIZE
+
+PROCESS_DUP_HANDLE = 0x0040
+
 CREATE_NEW        = 1
 CREATE_ALWAYS     = 2
 OPEN_EXISTING     = 3
diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -8,3 +8,7 @@
 .. branch: fix-literal-prev_digit-underscore
 
 Fix parsing for converting strings with underscore into ints
+
+.. branch: winmultiprocessing
+
+Improve multiprocessing support on win32
diff --git a/pypy/module/_multiprocessing/__init__.py b/pypy/module/_multiprocessing/__init__.py
--- a/pypy/module/_multiprocessing/__init__.py
+++ b/pypy/module/_multiprocessing/__init__.py
@@ -15,5 +15,5 @@
         interpleveldefs['closesocket'] = 'interp_win32_py3.multiprocessing_closesocket'
         interpleveldefs['recv'] = 'interp_win32_py3.multiprocessing_recv'
         interpleveldefs['send'] = 'interp_win32_py3.multiprocessing_send'
-    else:
-        interpleveldefs['sem_unlink'] = 'interp_semaphore.semaphore_unlink'
+    
+    interpleveldefs['sem_unlink'] = 'interp_semaphore.semaphore_unlink'
diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -36,8 +36,7 @@
         save_err=rffi.RFFI_SAVE_LASTERROR)
 
     def sem_unlink(name):
-        pass
-
+        return None
 else:
     from rpython.rlib import rposix
 
@@ -217,12 +216,13 @@
     def handle_w(space, w_handle):
         return rffi.cast(SEM_T, space.int_w(w_handle))
 
-    def semaphore_unlink(space, w_name):
-        name = space.text_w(w_name)
-        try:
-            sem_unlink(name)
-        except OSError as e:
-            raise wrap_oserror(space, e)
+# utilized by POSIX and win32
+def semaphore_unlink(space, w_name):
+    name = space.text_w(w_name)
+    try:
+        sem_unlink(name)
+    except OSError as e:
+        raise wrap_oserror(space, e)
 
 class CounterState:
     def __init__(self, space):
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -16,6 +16,7 @@
     GetSetProperty, TypeDef, generic_new_descr, interp_attrproperty,
     make_weakref_descr)
 
+_WIN32 = sys.platform.startswith('win')
 
 # XXX Hack to separate rpython and pypy
 def addr_as_object(addr, fd, space):
@@ -197,13 +198,29 @@
             self.register_finalizer(space)
 
     @unwrap_spec(family=int, type=int, proto=int,
-                 w_fileno=WrappedDefault(None))
+                 w_fdobj=WrappedDefault(None))
     def descr_init(self, space, family=AF_INET, type=SOCK_STREAM, proto=0,
-                   w_fileno=None):
+                   w_fdobj=None):
         try:
-            if not space.is_w(w_fileno, space.w_None):
-                sock = RSocket(family, type, proto,
-                               fd=space.c_filedescriptor_w(w_fileno))
+            if not space.is_w(w_fdobj, space.w_None):
+                if _WIN32 and space.isinstance_w(w_fdobj, space.w_bytes):
+                    from rpython.rlib.rsocket import _c
+                    # it is possible to pass some bytes representing a socket
+                    # in the file descriptor object on winodws
+                    fdobj = space.bytes_w(w_fdobj)
+                    info_charptr = rffi.str2charp(fdobj)
+                    try:
+                        info_ptr = rffi.cast(lltype.Ptr(_c.WSAPROTOCOL_INFOW), info_charptr)
+                        fd = _c.WSASocketW(_c.FROM_PROTOCOL_INFO, _c.FROM_PROTOCOL_INFO,
+                        _c.FROM_PROTOCOL_INFO, info_ptr, 0, _c.WSA_FLAG_OVERLAPPED)
+                        if fd == rsocket.INVALID_SOCKET:
+                            raise converted_error(space, rsocket.last_error())
+                        sock = RSocket(info_ptr.c_iAddressFamily, info_ptr.c_iSocketType, info_ptr.c_iProtocol, fd)
+                    finally:
+                        lltype.free(info_charptr, flavor='raw')
+                else:
+                    sock = RSocket(family, type, proto,
+                                   fd=space.c_filedescriptor_w(w_fdobj))
             else:
                 sock = RSocket(family, type, proto, inheritable=False)
             W_Socket.__init__(self, space, sock)
@@ -757,6 +774,26 @@
         finally:
             lltype.free(recv_ptr, flavor='raw')
 
+    @unwrap_spec(processid=int)
+    def share_w(self, space, processid):
+        from rpython.rtyper.lltypesystem import rffi, lltype
+        from rpython.rlib import rwin32
+        from rpython.rlib.rsocket import _c
+        info_ptr = lltype.malloc(_c.WSAPROTOCOL_INFOW, flavor='raw')
+        try:
+            winprocessid = rffi.cast(rwin32.DWORD, processid)
+            res = _c.WSADuplicateSocketW(
+                        self.sock.fd, winprocessid, info_ptr)
+
+            if res < 0:
+                raise converted_error(space, rsocket.last_error())
+
+            bytes_ptr = rffi.cast(rffi.CCHARP, info_ptr)
+            w_bytes = space.newbytes(rffi.charpsize2str(bytes_ptr, rffi.sizeof(_c.WSAPROTOCOL_INFOW)))
+        finally:
+            lltype.free(info_ptr, flavor='raw')
+        return w_bytes
+			
     @unwrap_spec(how="c_int")
     def shutdown_w(self, space, how):
         """shutdown(flag)
@@ -890,6 +927,7 @@
 """.split()
 if hasattr(rsocket._c, 'WSAIoctl'):
     socketmethodnames.append('ioctl')
+    socketmethodnames.append('share')
 if rsocket._c.HAVE_SENDMSG:
     socketmethodnames.append('sendmsg')
     socketmethodnames.append('recvmsg')
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -575,6 +575,26 @@
         raises(ValueError, s.ioctl, -1, None)
         s.ioctl(_socket.SIO_KEEPALIVE_VALS, (1, 100, 100))
 
+    def test_socket_sharelocal(self):
+        import _socket, sys, os
+        if sys.platform != 'win32':
+            skip("win32 only")
+        assert hasattr(_socket.socket, 'share')
+        s = _socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        s.listen()
+        data = s.share(os.getpid())
+        s2 = socket.fromshare(data)
+        try:
+            assert s.gettimeout() == s2.gettimeout()
+            assert s.family == s2.family
+            assert s.type == s2.type
+            if s.proto != 0:
+                assert s.proto == s2.proto
+        finally:
+            s.close()
+            s2.close()
+
+
     def test_dup(self):
         import _socket as socket, os
         s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py
--- a/rpython/rlib/_rsocket_rffi.py
+++ b/rpython/rlib/_rsocket_rffi.py
@@ -203,6 +203,7 @@
 FD_CONNECT_BIT FD_CLOSE_BIT
 WSA_IO_PENDING WSA_IO_INCOMPLETE WSA_INVALID_HANDLE
 WSA_INVALID_PARAMETER WSA_NOT_ENOUGH_MEMORY WSA_OPERATION_ABORTED
+WSA_FLAG_OVERLAPPED
 SIO_RCVALL SIO_KEEPALIVE_VALS
 
 SIOCGIFNAME SIOCGIFINDEX
@@ -1010,6 +1011,7 @@
     CConfig.WSAPROTOCOL_INFO = platform.Struct(
         'WSAPROTOCOL_INFOA',
         [])  # Struct is just passed between functions
+	   
     CConfig.FROM_PROTOCOL_INFO = platform.DefinedConstantInteger(
         'FROM_PROTOCOL_INFO')
 
@@ -1034,6 +1036,45 @@
         [('onoff', rffi.ULONG),
          ('keepalivetime', rffi.ULONG),
          ('keepaliveinterval', rffi.ULONG)])
+	
+    CConfig.GUID = platform.Struct(
+		     'struct _GUID',
+			 [('Data1', rffi.UINT),
+             ('Data2', rffi.UINT),
+             ('Data3', rffi.UINT),
+             ('Data4', rffi.CFixedArray(rffi.UCHAR, 8))
+         ])
+		 
+    CConfig.WSAPROTOCOLCHAIN = platform.Struct(
+        'struct _WSAPROTOCOLCHAIN',
+        [('ChainLen', rffi.INT),
+         ('ChainEntries', rffi.CFixedArray(rffi.UINT, 7))])
+	
+    WSAPROTOCOLCHAIN = CConfig.WSAPROTOCOLCHAIN
+    GUID = CConfig.GUID 
+	
+    CConfig.WSAPROTOCOL_INFOW = platform.Struct(
+        'struct _WSAPROTOCOL_INFOW',
+        [('dwServiceFlags1', rffi.UINT),
+         ('dwServiceFlags2', rffi.UINT),
+         ('dwServiceFlags3', rffi.UINT),
+         ('dwServiceFlags4', rffi.UINT),
+         ('dwProviderFlags', rffi.UINT),
+         ('ProviderId', GUID),
+         ('dwCatalogEntryId', rffi.UINT),
+         ('ProtocolChain', WSAPROTOCOLCHAIN),
+         ('iVersion', rffi.INT),
+         ('iAddressFamily', rffi.INT),
+         ('iMaxSockAddr', rffi.INT),
+         ('iMinSockAddr', rffi.INT),
+         ('iSocketType', rffi.INT),
+         ('iProtocol', rffi.INT),
+         ('iProtocolMaxOffset', rffi.INT),
+         ('iNetworkByteOrder', rffi.INT),
+         ('iSecurityScheme', rffi.INT),
+         ('dwMessageSize', rffi.UINT),
+         ('dwProviderReserved', rffi.UINT),
+         ('szProtocol',  rffi.CFixedArray(rffi.UCHAR, 256))])
 
 
 class cConfig:
@@ -1337,6 +1378,20 @@
                          rffi.VOIDP, rwin32.DWORD,
                          rwin32.LPDWORD, rffi.VOIDP, rffi.VOIDP],
                         rffi.INT, save_err=SAVE_ERR)
+
+    WSAPROTOCOL_INFOW = cConfig.WSAPROTOCOL_INFOW
+
+    WSADuplicateSocketW = external('WSADuplicateSocketW',
+                                 [socketfd_type, rwin32.DWORD,
+                                  lltype.Ptr(WSAPROTOCOL_INFOW)],
+                                  rffi.INT, save_err=SAVE_ERR)
+
+    WSASocketW = external('WSASocketW',
+                         [rffi.INT, rffi.INT, rffi.INT,
+                          lltype.Ptr(WSAPROTOCOL_INFOW),
+                          rwin32.DWORD, rwin32.DWORD],
+                         socketfd_type, save_err=SAVE_ERR)
+								  
     tcp_keepalive = cConfig.tcp_keepalive
 
     WSAPROTOCOL_INFO = cConfig.WSAPROTOCOL_INFO

From pypy.commits at gmail.com  Mon May 27 01:14:45 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 26 May 2019 22:14:45 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: try to fix win32 _socket.share test
Message-ID: <5ceb7245.1c69fb81.82c85.604f@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r96696:b7cc70885b5f
Date: 2019-05-27 08:14 +0300
http://bitbucket.org/pypy/pypy/changeset/b7cc70885b5f/

Log:	try to fix win32 _socket.share test

diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -583,7 +583,8 @@
         s = _socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         s.listen()
         data = s.share(os.getpid())
-        s2 = socket.fromshare(data)
+        # emulate socket.fromshare
+        s2 = _socket.socket(0, 0, 0, data)
         try:
             assert s.gettimeout() == s2.gettimeout()
             assert s.family == s2.family

From pypy.commits at gmail.com  Mon May 27 01:54:30 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 26 May 2019 22:54:30 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix test for extra ops after
 shadowstack-issue2722
Message-ID: <5ceb7b96.1c69fb81.e93e9.3d0e@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r96697:597a4f90a1b1
Date: 2019-05-27 08:53 +0300
http://bitbucket.org/pypy/pypy/changeset/597a4f90a1b1/

Log:	fix test for extra ops after shadowstack-issue2722

diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -425,9 +425,11 @@
         setarrayitem_raw(i153, 0, i106, descr=...)
         p156 = getfield_gc_r(p48, descr=...)
         i158 = getfield_raw_i(..., descr=...)
+        i160 = int_sub(i158, 16)
+        setfield_raw(#, i160, descr=...)
         setfield_gc(p48, p49, descr=...)
         setfield_gc(p134, ConstPtr(null), descr=...)
-        i160 = int_lt(i158, 0)
+        i160 = int_lt(i160, 0)
         guard_false(i160, descr=...)
         jump(..., descr=...)
         """)

From pypy.commits at gmail.com  Mon May 27 05:36:56 2019
From: pypy.commits at gmail.com (andrewjlawrence)
Date: Mon, 27 May 2019 02:36:56 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: Fix test
Message-ID: <5cebafb8.1c69fb81.25208.8b17@mx.google.com>

Author: andrewjlawrence
Branch: py3.6
Changeset: r96698:9bf9185580c6
Date: 2019-05-27 10:35 +0100
http://bitbucket.org/pypy/pypy/changeset/9bf9185580c6/

Log:	Fix test

diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -580,7 +580,7 @@
         if sys.platform != 'win32':
             skip("win32 only")
         assert hasattr(_socket.socket, 'share')
-        s = _socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM)
         s.listen()
         data = s.share(os.getpid())
         # emulate socket.fromshare

From pypy.commits at gmail.com  Mon May 27 07:21:00 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 27 May 2019 04:21:00 -0700 (PDT)
Subject: [pypy-commit] pypy default: the virtualenv issue was resolved so
 now win32 uses venv/Scripts/pypy-c.exe
Message-ID: <5cebc81c.1c69fb81.46e55.af9a@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r96699:129d1eff1693
Date: 2019-05-27 09:51 +0300
http://bitbucket.org/pypy/pypy/changeset/129d1eff1693/

Log:	the virtualenv issue was resolved so now win32 uses
	venv/Scripts/pypy-c.exe

diff --git a/testrunner/get_info.py b/testrunner/get_info.py
--- a/testrunner/get_info.py
+++ b/testrunner/get_info.py
@@ -10,8 +10,7 @@
 BASE_DIR = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 if sys.platform.startswith('win'):
     TARGET_NAME = r'pypy-c.exe'
-    # see https://github.com/pypa/virtualenv/issues/789
-    TARGET_DIR = 'bin'
+    TARGET_DIR = 'Scripts'
 else:
     TARGET_NAME = 'pypy-c'
     TARGET_DIR = 'bin'

From pypy.commits at gmail.com  Mon May 27 07:21:02 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 27 May 2019 04:21:02 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: fix merge
Message-ID: <5cebc81e.1c69fb81.653f7.e6e0@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r96700:3ddfc6aee731
Date: 2019-05-27 14:18 +0300
http://bitbucket.org/pypy/pypy/changeset/3ddfc6aee731/

Log:	fix merge

diff --git a/lib_pypy/_cffi_ssl/_stdssl/error.py b/lib_pypy/_cffi_ssl/_stdssl/error.py
--- a/lib_pypy/_cffi_ssl/_stdssl/error.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/error.py
@@ -1,5 +1,6 @@
 import sys
 import os
+import socket
 import traceback
 from _pypy_openssl import ffi
 from _pypy_openssl import lib

From pypy.commits at gmail.com  Mon May 27 10:20:56 2019
From: pypy.commits at gmail.com (fijal)
Date: Mon, 27 May 2019 07:20:56 -0700 (PDT)
Subject: [pypy-commit] pypy default: Ignore strange entries
Message-ID: <5cebf248.1c69fb81.cf712.f161@mx.google.com>

Author: fijal
Branch: 
Changeset: r96701:06e38bb00f7e
Date: 2019-05-27 16:20 +0200
http://bitbucket.org/pypy/pypy/changeset/06e38bb00f7e/

Log:	Ignore strange entries

diff --git a/rpython/tool/jitlogparser/parser.py b/rpython/tool/jitlogparser/parser.py
--- a/rpython/tool/jitlogparser/parser.py
+++ b/rpython/tool/jitlogparser/parser.py
@@ -504,8 +504,10 @@
     for line in lines:
         if line:
             num, count = line.split(':', 2)
-            mapping[num].count = int(count)
-
+            try:
+                mapping[num].count = int(count)
+            except KeyError:
+                pass # too bad
 
 def mangle_descr(descr):
     if descr.startswith('TargetToken('):

From pypy.commits at gmail.com  Mon May 27 10:37:29 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 27 May 2019 07:37:29 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: fix merge
Message-ID: <5cebf629.1c69fb81.96e2c.9c71@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r96702:5c4489ede2be
Date: 2019-05-27 17:36 +0300
http://bitbucket.org/pypy/pypy/changeset/5c4489ede2be/

Log:	fix merge

diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -222,7 +222,7 @@
     def _new__ssl_socket(sslctx, sock, socket_type, server_hostname, inbio, outbio):
         self = _SSLSocket(sslctx)
         ctx = sslctx.ctx
-        self.owner = ssl_sock  # weakref
+        self.owner = None
 
         if server_hostname:
             self.server_hostname = server_hostname.decode('idna', 'strict')

From pypy.commits at gmail.com  Mon May 27 18:35:47 2019
From: pypy.commits at gmail.com (andrewjlawrence)
Date: Mon, 27 May 2019 15:35:47 -0700 (PDT)
Subject: [pypy-commit] pypy winconsoleio: Started implementing winconsoleio
Message-ID: <5cec6643.1c69fb81.ffcce.4343@mx.google.com>

Author: andrewjlawrence
Branch: winconsoleio
Changeset: r96703:01c16b67cba4
Date: 2019-05-27 10:27 +0100
http://bitbucket.org/pypy/pypy/changeset/01c16b67cba4/

Log:	Started implementing winconsoleio

diff --git a/pypy/module/_io/interp_win32consoleio.py b/pypy/module/_io/interp_win32consoleio.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_io/interp_win32consoleio.py
@@ -0,0 +1,19 @@
+import sys
+
+from pypy.module._io.interp_iobase import W_RawIOBase
+
+class W_WinConsoleIO(W_RawIOBase):
+    def __init__(self, space):
+        W_RawIOBase.__init__(self, space)
+
+    def descr_init(self, space, w_nameobj, w_mode="r", w_closefd=True, w_opener=None):
+        #self.fd = -1
+        #self.created = 0
+        #self.readable = 0
+        #self.writable = 0
+        #self.closehandle = 0;
+        #self.blksize = 0
+        fd = space.int_w(w_nameobj)
+        if self.fd < 0:
+            raise oefmt(space.w_ValueError, "negative file descriptor")
+        self.fd = fd
diff --git a/pypy/module/_io/test/test_win32consoleio.py b/pypy/module/_io/test/test_win32consoleio.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_io/test/test_win32consoleio.py
@@ -0,0 +1,12 @@
+class AppTestWinConsoleIO:
+    spaceconfig = dict(usemodules=['_io', '_locale', 'array'])
+
+    def setup_class(cls):
+        from rpython.rlib.rarithmetic import INT_MAX, UINT_MAX
+        space = cls.space
+        cls.w_INT_MAX = space.wrap(INT_MAX)
+        cls.w_UINT_MAX = space.wrap(UINT_MAX)
+
+    def test_constructor(self):
+        import _io
+        t = _io.WinConsoleIO()

From pypy.commits at gmail.com  Mon May 27 18:35:49 2019
From: pypy.commits at gmail.com (andrewjlawrence)
Date: Mon, 27 May 2019 15:35:49 -0700 (PDT)
Subject: [pypy-commit] pypy winconsoleio: Add winconsoleio to init
Message-ID: <5cec6645.1c69fb81.fee0f.aad7@mx.google.com>

Author: andrewjlawrence
Branch: winconsoleio
Changeset: r96704:d1f9ab0fd01b
Date: 2019-05-27 21:06 +0100
http://bitbucket.org/pypy/pypy/changeset/d1f9ab0fd01b/

Log:	Add winconsoleio to init

diff --git a/pypy/module/_io/__init__.py b/pypy/module/_io/__init__.py
--- a/pypy/module/_io/__init__.py
+++ b/pypy/module/_io/__init__.py
@@ -23,6 +23,7 @@
         'BufferedRWPair': 'interp_bufferedio.W_BufferedRWPair',
         'BufferedRandom': 'interp_bufferedio.W_BufferedRandom',
         'TextIOWrapper': 'interp_textio.W_TextIOWrapper',
+        'WindowsConsoleIO': 'interp_win32consoleio.W_WinConsoleIO',
 
         'open': 'interp_io.open',
         'IncrementalNewlineDecoder': 'interp_textio.W_IncrementalNewlineDecoder',

From pypy.commits at gmail.com  Mon May 27 18:35:51 2019
From: pypy.commits at gmail.com (andrewjlawrence)
Date: Mon, 27 May 2019 15:35:51 -0700 (PDT)
Subject: [pypy-commit] pypy winconsoleio: Merged py3.6
Message-ID: <5cec6647.1c69fb81.28c56.8a7e@mx.google.com>

Author: andrewjlawrence
Branch: winconsoleio
Changeset: r96705:617ba4ae9ec9
Date: 2019-05-27 21:07 +0100
http://bitbucket.org/pypy/pypy/changeset/617ba4ae9ec9/

Log:	Merged py3.6

diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
@@ -40,6 +40,11 @@
  */
 static const long Cryptography_HAS_OP_NO_COMPRESSION;
 
+/* Internally invented symbol to tell us if SSL_OP_ENABLE_MIDDLEBOX_COMPAT is
+ * supported
+ */
+static const long Cryptography_HAS_OP_ENABLE_MIDDLEBOX_COMPAT;
+
 static const long Cryptography_HAS_SSL_OP_MSIE_SSLV2_RSA_PADDING;
 static const long Cryptography_HAS_SSL_SET_SSL_CTX;
 static const long Cryptography_HAS_SSL_OP_NO_TICKET;
@@ -73,6 +78,7 @@
 static const long SSL_OP_NETSCAPE_REUSE_CIPHER_CHANGE_BUG;
 static const long SSL_OP_NO_SSLv2;
 static const long SSL_OP_SSLREF2_REUSE_CERT_TYPE_BUG;
+static const long SSL_OP_ENABLE_MIDDLEBOX_COMPAT;
 static const long SSL_OP_MICROSOFT_BIG_SSLV3_BUFFER;
 static const long SSL_OP_MSIE_SSLV2_RSA_PADDING;
 static const long SSL_OP_SSLEAY_080_CLIENT_DH_BUG;
@@ -560,6 +566,13 @@
 const long SSL_OP_NO_COMPRESSION = 0;
 #endif
 
+#ifdef SSL_OP_ENABLE_MIDDLEBOX_COMPAT
+static const long Cryptography_HAS_OP_ENABLE_MIDDLEBOX_COMPAT = 1;
+#else
+static const long Cryptography_HAS_OP_ENABLE_MIDDLEBOX_COMPAT = 0;
+const long SSL_OP_ENABLE_MIDDLEBOX_COMPAT = 0;
+#endif
+
 #ifdef SSL_OP_NO_TLSv1_1
 static const long Cryptography_HAS_TLSv1_1 = 1;
 #else
diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -20,6 +20,7 @@
         pyerr_write_unraisable)
 from _cffi_ssl._stdssl import error
 from select import select
+import socket
 from enum import IntEnum as _IntEnum
 
 if sys.platform == 'win32':
@@ -221,6 +222,7 @@
     def _new__ssl_socket(sslctx, sock, socket_type, server_hostname, inbio, outbio):
         self = _SSLSocket(sslctx)
         ctx = sslctx.ctx
+        self.owner = None
 
         if server_hostname:
             self.server_hostname = server_hostname.decode('idna', 'strict')
@@ -289,7 +291,8 @@
     def owner(self, value):
         if value is None:
             self._owner = None
-        self._owner = weakref.ref(value)
+        else:
+            self._owner = weakref.ref(value)
 
     @property
     def context(self):
@@ -311,9 +314,6 @@
         return self.socket_type == SSL_SERVER
 
     def do_handshake(self):
-        # delay to prevent circular imports
-        import socket
-
         sock = self.get_socket_or_connection_gone()
         ssl = self.ssl
         timeout = _socket_timeout(sock)
@@ -385,9 +385,6 @@
                 return _decode_certificate(self.peer_cert)
 
     def write(self, bytestring):
-        # delay to prevent circular imports
-        import socket
-
         deadline = 0
         b = _str_to_ffi_buffer(bytestring)
         sock = self.get_socket_or_connection_gone()
@@ -446,9 +443,6 @@
             raise pyssl_error(self, length)
 
     def read(self, length, buffer_into=None):
-        # delay to prevent circular imports
-        import socket
-
         ssl = self.ssl
 
         if length < 0 and buffer_into is None:
@@ -589,9 +583,6 @@
         return sock
 
     def shutdown(self):
-        # delay to prevent circular imports
-        import socket
-
         sock = self.get_socket_or_None()
         nonblocking = False
         ssl = self.ssl
@@ -1068,7 +1059,7 @@
             self._add_ca_certs(buf, len(buf), ca_file_type)
 
         # load cafile or capath
-        if cafile or capath:
+        if cafile is not None or capath is not None:
             if cafile is None:
                 cafilebuf = ffi.NULL
             else:
@@ -1577,3 +1568,5 @@
                            "enough data to seed the PRNG");
         return bytecount
 
+socket.RAND_add = RAND_add
+socket.RAND_status = RAND_status
diff --git a/lib_pypy/_cffi_ssl/_stdssl/error.py b/lib_pypy/_cffi_ssl/_stdssl/error.py
--- a/lib_pypy/_cffi_ssl/_stdssl/error.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/error.py
@@ -1,5 +1,6 @@
 import sys
 import os
+import socket
 import traceback
 from _pypy_openssl import ffi
 from _pypy_openssl import lib
@@ -26,6 +27,13 @@
         if self.strerror and isinstance(self.strerror, str):
             return self.strerror
         return str(self.args)
+# these are expected on socket as well
+socket.sslerror = SSLError
+for v in [ 'SSL_ERROR_ZERO_RETURN', 'SSL_ERROR_WANT_READ',
+     'SSL_ERROR_WANT_WRITE', 'SSL_ERROR_WANT_X509_LOOKUP', 'SSL_ERROR_SYSCALL',
+     'SSL_ERROR_SSL', 'SSL_ERROR_WANT_CONNECT', 'SSL_ERROR_EOF',
+     'SSL_ERROR_INVALID_ERROR_CODE' ]:
+    setattr(socket, v, locals()[v]) 
 
 class SSLZeroReturnError(SSLError):
     """ SSL/TLS session closed cleanly. """
diff --git a/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py b/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py
--- a/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py
@@ -1,101 +1,101 @@
 from _pypy_openssl import lib, ffi
-
-
-def enum_certificates(store_name):
-    """Retrieve certificates from Windows' cert store.
-
-store_name may be one of 'CA', 'ROOT' or 'MY'.  The system may provide
-more cert storages, too.  The function returns a list of (bytes,
-encoding_type, trust) tuples.  The encoding_type flag can be interpreted
-with X509_ASN_ENCODING or PKCS_7_ASN_ENCODING. The trust setting is either
-a set of OIDs or the boolean True.
-    """
-    hStore = lib.CertOpenStore(lib.CERT_STORE_PROV_SYSTEM_A, 0, ffi.NULL,
-                               lib.CERT_STORE_READONLY_FLAG | lib.CERT_SYSTEM_STORE_LOCAL_MACHINE,
-                               bytes(store_name, "ascii"))
-    if hStore == ffi.NULL:
-        raise WindowsError(*ffi.getwinerror())
-    
-    result = []
-    pCertCtx = ffi.NULL
-    try:
-        while True:
-            pCertCtx = lib.CertEnumCertificatesInStore(hStore, pCertCtx)
-            if pCertCtx == ffi.NULL:
-                break
-            cert = ffi.buffer(pCertCtx.pbCertEncoded, pCertCtx.cbCertEncoded)[:]
-            enc = certEncodingType(pCertCtx.dwCertEncodingType)
-            keyusage = parseKeyUsage(pCertCtx, lib.CERT_FIND_PROP_ONLY_ENHKEY_USAGE_FLAG)
-            if keyusage is True:
-                keyusage = parseKeyUsage(pCertCtx, lib.CERT_FIND_EXT_ONLY_ENHKEY_USAGE_FLAG)
-            result.append((cert, enc, keyusage))
-    finally:
-        if pCertCtx != ffi.NULL:
-            lib.CertFreeCertificateContext(pCertCtx)
-        if not lib.CertCloseStore(hStore, 0):
-            # This error case might shadow another exception.
-            raise WindowsError(*ffi.getwinerror())
-    return result
-
-
-def enum_crls(store_name):
-    """Retrieve CRLs from Windows' cert store.
-
-store_name may be one of 'CA', 'ROOT' or 'MY'.  The system may provide
-more cert storages, too.  The function returns a list of (bytes,
-encoding_type) tuples.  The encoding_type flag can be interpreted with
-X509_ASN_ENCODING or PKCS_7_ASN_ENCODING."""
-    hStore = lib.CertOpenStore(lib.CERT_STORE_PROV_SYSTEM_A, 0, ffi.NULL,
-                               lib.CERT_STORE_READONLY_FLAG | lib.CERT_SYSTEM_STORE_LOCAL_MACHINE,
-                               bytes(store_name, "ascii"))
-    if hStore == ffi.NULL:
-        raise WindowsError(*ffi.getwinerror())
-
-    result = []
-    pCrlCtx = ffi.NULL
-    try:
-        while True:
-            pCrlCtx = lib.CertEnumCRLsInStore(hStore, pCrlCtx)
-            if pCrlCtx == ffi.NULL:
-                break
-            crl = ffi.buffer(pCrlCtx.pbCrlEncoded, pCrlCtx.cbCrlEncoded)[:]
-            enc = certEncodingType(pCrlCtx.dwCertEncodingType)
-            result.append((crl, enc))
-    finally:
-        if pCrlCtx != ffi.NULL:
-            lib.CertFreeCRLContext(pCrlCtx)
-        if not lib.CertCloseStore(hStore, 0):
-            # This error case might shadow another exception.
-            raise WindowsError(*ffi.getwinerror())
-    return result
-
-
-def certEncodingType(encodingType):
-    if encodingType == lib.X509_ASN_ENCODING:
-        return "x509_asn"
-    if encodingType == lib.PKCS_7_ASN_ENCODING:
-        return "pkcs_7_asn"
-    return encodingType
-
-def parseKeyUsage(pCertCtx, flags):
-    pSize = ffi.new("DWORD *")
-    if not lib.CertGetEnhancedKeyUsage(pCertCtx, flags, ffi.NULL, pSize):
-        error_with_message = ffi.getwinerror()
-        if error_with_message[0] == lib.CRYPT_E_NOT_FOUND:
-            return True
-        raise WindowsError(*error_with_message)
-
-    pUsageMem = ffi.new("char[]", pSize[0])
-    pUsage = ffi.cast("PCERT_ENHKEY_USAGE", pUsageMem)
-    if not lib.CertGetEnhancedKeyUsage(pCertCtx, flags, pUsage, pSize):
-        error_with_message = ffi.getwinerror()
-        if error_with_message[0] == lib.CRYPT_E_NOT_FOUND:
-            return True
-        raise WindowsError(*error_with_message)
-
-    retval = set()
-    for i in range(pUsage.cUsageIdentifier):
-        if pUsage.rgpszUsageIdentifier[i]:
-            oid = ffi.string(pUsage.rgpszUsageIdentifier[i]).decode('ascii')
-            retval.add(oid)
-    return retval
+
+
+def enum_certificates(store_name):
+    """Retrieve certificates from Windows' cert store.
+
+store_name may be one of 'CA', 'ROOT' or 'MY'.  The system may provide
+more cert storages, too.  The function returns a list of (bytes,
+encoding_type, trust) tuples.  The encoding_type flag can be interpreted
+with X509_ASN_ENCODING or PKCS_7_ASN_ENCODING. The trust setting is either
+a set of OIDs or the boolean True.
+    """
+    hStore = lib.CertOpenStore(lib.CERT_STORE_PROV_SYSTEM_A, 0, ffi.NULL,
+                               lib.CERT_STORE_READONLY_FLAG | lib.CERT_SYSTEM_STORE_LOCAL_MACHINE,
+                               bytes(store_name, "ascii"))
+    if hStore == ffi.NULL:
+        raise WindowsError(*ffi.getwinerror())
+    
+    result = []
+    pCertCtx = ffi.NULL
+    try:
+        while True:
+            pCertCtx = lib.CertEnumCertificatesInStore(hStore, pCertCtx)
+            if pCertCtx == ffi.NULL:
+                break
+            cert = ffi.buffer(pCertCtx.pbCertEncoded, pCertCtx.cbCertEncoded)[:]
+            enc = certEncodingType(pCertCtx.dwCertEncodingType)
+            keyusage = parseKeyUsage(pCertCtx, lib.CERT_FIND_PROP_ONLY_ENHKEY_USAGE_FLAG)
+            if keyusage is True:
+                keyusage = parseKeyUsage(pCertCtx, lib.CERT_FIND_EXT_ONLY_ENHKEY_USAGE_FLAG)
+            result.append((cert, enc, keyusage))
+    finally:
+        if pCertCtx != ffi.NULL:
+            lib.CertFreeCertificateContext(pCertCtx)
+        if not lib.CertCloseStore(hStore, 0):
+            # This error case might shadow another exception.
+            raise WindowsError(*ffi.getwinerror())
+    return result
+
+
+def enum_crls(store_name):
+    """Retrieve CRLs from Windows' cert store.
+
+store_name may be one of 'CA', 'ROOT' or 'MY'.  The system may provide
+more cert storages, too.  The function returns a list of (bytes,
+encoding_type) tuples.  The encoding_type flag can be interpreted with
+X509_ASN_ENCODING or PKCS_7_ASN_ENCODING."""
+    hStore = lib.CertOpenStore(lib.CERT_STORE_PROV_SYSTEM_A, 0, ffi.NULL,
+                               lib.CERT_STORE_READONLY_FLAG | lib.CERT_SYSTEM_STORE_LOCAL_MACHINE,
+                               bytes(store_name, "ascii"))
+    if hStore == ffi.NULL:
+        raise WindowsError(*ffi.getwinerror())
+
+    result = []
+    pCrlCtx = ffi.NULL
+    try:
+        while True:
+            pCrlCtx = lib.CertEnumCRLsInStore(hStore, pCrlCtx)
+            if pCrlCtx == ffi.NULL:
+                break
+            crl = ffi.buffer(pCrlCtx.pbCrlEncoded, pCrlCtx.cbCrlEncoded)[:]
+            enc = certEncodingType(pCrlCtx.dwCertEncodingType)
+            result.append((crl, enc))
+    finally:
+        if pCrlCtx != ffi.NULL:
+            lib.CertFreeCRLContext(pCrlCtx)
+        if not lib.CertCloseStore(hStore, 0):
+            # This error case might shadow another exception.
+            raise WindowsError(*ffi.getwinerror())
+    return result
+
+
+def certEncodingType(encodingType):
+    if encodingType == lib.X509_ASN_ENCODING:
+        return "x509_asn"
+    if encodingType == lib.PKCS_7_ASN_ENCODING:
+        return "pkcs_7_asn"
+    return encodingType
+
+def parseKeyUsage(pCertCtx, flags):
+    pSize = ffi.new("DWORD *")
+    if not lib.CertGetEnhancedKeyUsage(pCertCtx, flags, ffi.NULL, pSize):
+        error_with_message = ffi.getwinerror()
+        if error_with_message[0] == lib.CRYPT_E_NOT_FOUND:
+            return True
+        raise WindowsError(*error_with_message)
+
+    pUsageMem = ffi.new("char[]", pSize[0])
+    pUsage = ffi.cast("PCERT_ENHKEY_USAGE", pUsageMem)
+    if not lib.CertGetEnhancedKeyUsage(pCertCtx, flags, pUsage, pSize):
+        error_with_message = ffi.getwinerror()
+        if error_with_message[0] == lib.CRYPT_E_NOT_FOUND:
+            return True
+        raise WindowsError(*error_with_message)
+
+    retval = set()
+    for i in range(pUsage.cUsageIdentifier):
+        if pUsage.rgpszUsageIdentifier[i]:
+            oid = ffi.string(pUsage.rgpszUsageIdentifier[i]).decode('ascii')
+            retval.add(oid)
+    return retval
diff --git a/lib_pypy/_overlapped.py b/lib_pypy/_overlapped.py
--- a/lib_pypy/_overlapped.py
+++ b/lib_pypy/_overlapped.py
@@ -173,7 +173,6 @@
         olderr = _kernel32.GetLastError()
         hascompletedio = HasOverlappedIoCompleted(self.overlapped[0])      
         if not hascompletedio and self.type != OverlappedType.TYPE_NOT_STARTED:
-            
             wait = _kernel32.CancelIoEx(self.handle, self.overlapped)
             ret = self.GetOverlappedResult(wait)
             err = _winapi.ERROR_SUCCESS
@@ -186,6 +185,7 @@
                SetFromWindowsErr(err)
         if self.overlapped[0].hEvent != 0:
             _winapi.CloseHandle(self.overlapped[0].hEvent)
+        _winapi.SetLastError(olderr)
 
     @property
     def event(self):
@@ -216,10 +216,6 @@
         else:
             return transferred[0]
 
-    def getbuffer(self):
-        xxx
-        return None
-
     def cancel(self):
         result = True
         if self.type == OverlappedType.TYPE_NOT_STARTED or self.type == OverlappedType.TYPE_WAIT_NAMED_PIPE_AND_CONNECT:
diff --git a/lib_pypy/_pypy_testcapi.py b/lib_pypy/_pypy_testcapi.py
--- a/lib_pypy/_pypy_testcapi.py
+++ b/lib_pypy/_pypy_testcapi.py
@@ -64,7 +64,6 @@
 
     compiler = new_compiler()
     compiler.output_dir = output_dir
-
     # Compile .c file
     include_dir = os.path.join(thisdir, '..', 'include')
     if sys.platform == 'win32':
@@ -80,7 +79,7 @@
     output_filename = modulename + _get_c_extension_suffix()
     if sys.platform == 'win32':
         libname = 'python{0[0]}{0[1]}'.format(sys.version_info)
-        library = os.path.join(thisdir, '..', 'lib', libname)
+        library = os.path.join(thisdir, '..', 'libs', libname)
         if not os.path.exists(library + '.lib'):
             # For a local translation or nightly build
             library = os.path.join(thisdir, '..', 'pypy', 'goal', libname)
diff --git a/lib_pypy/_pypy_winbase_build.py b/lib_pypy/_pypy_winbase_build.py
--- a/lib_pypy/_pypy_winbase_build.py
+++ b/lib_pypy/_pypy_winbase_build.py
@@ -101,8 +101,11 @@
 HANDLE WINAPI CreateFileW(LPCWSTR, DWORD, DWORD, LPSECURITY_ATTRIBUTES,
                    DWORD, DWORD, HANDLE);
 BOOL ReadFile(HANDLE, LPVOID, DWORD, LPDWORD, LPOVERLAPPED);
+BOOL WaitNamedPipeA(LPCSTR, DWORD);
+BOOL WINAPI WriteFile(HANDLE, LPCVOID, DWORD, LPDWORD, LPOVERLAPPED);
 BOOL WINAPI SetNamedPipeHandleState(HANDLE, LPDWORD, LPDWORD, LPDWORD);
 BOOL WINAPI ConnectNamedPipe(HANDLE, LPOVERLAPPED);
+BOOL WINAPI PeekNamedPipe(HANDLE, LPVOID, DWORD, LPDWORD, LPDWORD, LPDWORD);
 HANDLE WINAPI CreateEventA(LPSECURITY_ATTRIBUTES, BOOL, BOOL, LPCSTR);
 HANDLE WINAPI CreateEventW(LPSECURITY_ATTRIBUTES, BOOL, BOOL, LPCWSTR);
 BOOL WINAPI SetEvent(HANDLE);
@@ -110,8 +113,11 @@
 BOOL WINAPI CancelIoEx(HANDLE, LPOVERLAPPED);
 BOOL WINAPI CloseHandle(HANDLE);
 DWORD WINAPI GetLastError(VOID);
+void WINAPI SetLastError(DWORD);
 BOOL WINAPI GetOverlappedResult(HANDLE, LPOVERLAPPED, LPDWORD, BOOL);
 HANDLE WINAPI GetCurrentProcess(void);
+HANDLE OpenProcess(DWORD, BOOL, DWORD);
+void ExitProcess(UINT);
 BOOL WINAPI DuplicateHandle(HANDLE, HANDLE, HANDLE, LPHANDLE,
                             DWORD, BOOL, DWORD);
 BOOL WINAPI CreateProcessA(char *, char *, void *,
@@ -147,7 +153,7 @@
 BOOL WINAPI GetQueuedCompletionStatus(HANDLE, LPDWORD, ULONG**, LPOVERLAPPED*, DWORD);
 HANDLE WINAPI CreateIoCompletionPort(HANDLE, HANDLE, ULONG_PTR, DWORD);
 
-BOOL WINAPI WriteFile(HANDLE, LPCVOID, DWORD, LPDWORD, LPOVERLAPPED);
+
 
 #define WT_EXECUTEINWAITTHREAD 0x00000004
 #define WT_EXECUTEONLYONCE 0x00000008
diff --git a/lib_pypy/_pypy_winbase_cffi.py b/lib_pypy/_pypy_winbase_cffi.py
--- a/lib_pypy/_pypy_winbase_cffi.py
+++ b/lib_pypy/_pypy_winbase_cffi.py
@@ -3,8 +3,8 @@
 
 ffi = _cffi_backend.FFI('_pypy_winbase_cffi',
     _version = 0x2601,
-    _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x68\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x01\x42\x03\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x1A\x03\x00\x01\x3B\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x4C\x03\x00\x00\x27\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x27\x11\x00\x00\x27\x11\x00\x01\x47\x03\x00\x01\x3C\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x03\x00\x00\x33\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x33\x11\x00\x00\x11\x11\x00\x01\x32\x03\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x22\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x22\x11\x00\x00\x21\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x22\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x21\x11\x00\x00\x22\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x01\x48\x03\x00\x00\x0A\x01\x00\x00\x21\x11\x00\x00\x21\x11\x00\x00\x22\x11\x00\x01\x2D\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x5E\x11\x00\x00\x0A\x01\x00\x00\x21\x11\x00\x00\x0A\x01\x00\x00\x22\x11\x00\x00\x63\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x21\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x21\x11\x00\x00\x21\x03\x00\x00\x22\x03\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x21\x11\x00\x00\x21\x11\x00\x00\x21\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x22\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x21\x11\x00\x00\x22\x11\x00\x00\x63\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x21\x11\x00\x00\x22\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x33\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x01\x68\x03\x00\x00\x0A\x01\x00\x00\x21\x11\x00\x00\x22\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\xE2\x03\x00\x00\x07\x01\x00\x01\x4B\x03\x00\x00\x15\x11\x00\x00\x01\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\xB3\x11\x00\x00\xB3\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\xB3\x11\x00\x00\xB3\x11\x00\x00\x2F\x11\x00\x00\x30\x11\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x70\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x1A\x0D\x00\x00\x0A\x01\x00\x00\x33\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x1A\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x1A\x0D\x00\x00\x11\x11\x00\x00\xB3\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x1A\x0D\x00\x00\x02\x0F\x00\x00\xDD\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\xDD\x0D\x00\x00\x00\x0F\x00\x00\xDD\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x11\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x11\x0D\x00\x00\x00\x0F\x00\x00\x11\x0D\x00\x01\x41\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x01\x4C\x03\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xEC\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xE2\x03\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xEF\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xEC\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xEF\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xEC\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xB3\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xEC\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xF5\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xEC\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x01\x68\x0D\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x00\x0F\x00\x01\x68\x0D\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x0C\x09\x00\x01\x38\x03\x00\x00\x13\x09\x00\x01\x3A\x03\x00\x00\x14\x09\x00\x00\x0D\x09\x00\x00\x09\x09\x00\x01\x3E\x03\x00\x00\x0E\x09\x00\x01\x40\x03\x00\x00\x0A\x09\x00\x00\x0F\x09\x00\x00\x15\x09\x00\x01\x46\x03\x00\x01\x45\x03\x00\x00\x17\x09\x00\x00\x16\x09\x00\x00\x0B\x09\x00\x00\x10\x09\x00\x01\x4A\x03\x00\x00\x11\x09\x00\x00\x12\x09\x00\x00\x02\x01\x00\x01\x4C\x05\x00\x00\x00\x0E\x00\x01\x4C\x05\x00\x00\x00\x08\x00\x00\x4D\x03\x00\x00\x53\x03\x00\x00\x98\x03\x00\x00\x05\x01\x00\x00\x01\x09\x00\x00\x04\x09\x00\x00\x07\x09\x00\x00\x08\x09\x00\x00\x00\x09\x00\x00\x02\x09\x00\x00\x03\x09\x00\x00\x05\x09\x00\x00\x06\x09\x00\x01\x5F\x03\x00\x00\x04\x01\x00\x01\x5F\x05\x00\x00\x00\x10\x00\x01\x5F\x05\x00\x00\x00\x08\x00\x00\x1A\x05\x00\x00\x00\x07\x00\x00\xDD\x05\x00\x00\x00\x08\x00\x00\x00\x01\x00\x00\xE2\x05\x00\x00\x01\x00',
-    _globals = (b'\x00\x00\x40\x23CancelIo',0,b'\x00\x00\x43\x23CancelIoEx',0,b'\x00\x00\x40\x23CloseHandle',0,b'\x00\x00\x43\x23ConnectNamedPipe',0,b'\x00\x00\xEB\x23CreateEventA',0,b'\x00\x00\xF1\x23CreateEventW',0,b'\x00\x00\xF7\x23CreateFileA',0,b'\x00\x01\x24\x23CreateFileW',0,b'\x00\x01\x12\x23CreateIoCompletionPort',0,b'\x00\x01\x00\x23CreateNamedPipeA',0,b'\x00\x01\x1A\x23CreateNamedPipeW',0,b'\x00\x00\x32\x23CreatePipe',0,b'\x00\x00\x26\x23CreateProcessA',0,b'\x00\x00\xB9\x23CreateProcessW',0,b'\x00\x00\xA2\x23DuplicateHandle',0,b'\x00\x01\x18\x23GetCurrentProcess',0,b'\x00\x00\x72\x23GetExitCodeProcess',0,b'\x00\x00\xDA\x23GetLastError',0,b'\x00\x00\xD5\x23GetModuleFileNameW',0,b'\x00\x00\x47\x23GetOverlappedResult',0,b'\x00\x00\xE9\x23GetProcessHeap',0,b'\x00\x00\x76\x23GetQueuedCompletionStatus',0,b'\x00\x01\x0F\x23GetStdHandle',0,b'\x00\x00\xDA\x23GetVersion',0,b'\x00\x00\xE4\x23HeapAlloc',0,b'\x00\x00\x18\x23HeapFree',0,b'\xFF\xFF\xFF\x1FMAX_PROTOCOL_CHAIN',7,b'\x00\x00\x83\x23PostQueuedCompletionStatus',0,b'\x00\x00\x1D\x23ReadFile',0,b'\x00\x00\x38\x23RegisterWaitForSingleObject',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\xC8\x23SetErrorMode',0,b'\x00\x00\x40\x23SetEvent',0,b'\x00\x00\x7D\x23SetNamedPipeHandleState',0,b'\x00\x00\x6E\x23TerminateProcess',0,b'\x00\x00\x40\x23UnregisterWait',0,b'\x00\x00\x94\x23UnregisterWaitEx',0,b'\x00\x00\x89\x23WSAIoctl',0,b'\xFF\xFF\xFF\x1FWSAPROTOCOL_LEN',255,b'\x00\x00\x5C\x23WSARecv',0,b'\x00\x00\x65\x23WSASend',0,b'\x00\x00\xB2\x23WSAStringToAddressW',0,b'\xFF\xFF\xFF\x1FWT_EXECUTEINWAITTHREAD',4,b'\xFF\xFF\xFF\x1FWT_EXECUTEONLYONCE',8,b'\x00\x00\xCB\x23WaitForMultipleObjects',0,b'\x00\x00\xD1\x23WaitForSingleObject',0,b'\x00\x00\xAB\x23WriteFile',0,b'\x00\x00\xC5\x23_get_osfhandle',0,b'\x00\x00\x24\x23_getch',0,b'\x00\x00\x24\x23_getche',0,b'\x00\x00\xDF\x23_getwch',0,b'\x00\x00\xDF\x23_getwche',0,b'\x00\x00\x24\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\xE1\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\xDC\x23_ungetwch',0,b'\x00\x00\x13\x23bind',0,b'\x00\x00\x10\x23closesocket',0,b'\x00\x00\xDC\x23htons',0,b'\x00\x01\x0A\x23socket',0),
-    _struct_unions = ((b'\x00\x00\x01\x59\x00\x00\x00\x03$1',b'\x00\x01\x55\x11DUMMYSTRUCTNAME',b'\x00\x00\x11\x11Pointer'),(b'\x00\x00\x01\x55\x00\x00\x00\x02$2',b'\x00\x00\x1A\x11Offset',b'\x00\x00\x1A\x11OffsetHigh'),(b'\x00\x00\x01\x5A\x00\x00\x00\x03$3',b'\x00\x01\x60\x11Byte',b'\x00\x01\x66\x11Word'),(b'\x00\x00\x01\x5B\x00\x00\x00\x01$4',b'\x00\x01\x56\x11',b'\x00\x00\x1A\x11Value'),(b'\x00\x00\x01\x56\x00\x00\x00\x02$5',b'\x00\x00\x1A\x13\x00\x00\x00\x1CZone',b'\x00\x00\x1A\x13\x00\x00\x00\x04Level'),(b'\x00\x00\x01\x5C\x00\x00\x00\x03$6',b'\x00\x00\x1A\x11sin6_scope_id',b'\x00\x01\x40\x11sin6_scope_struct'),(b'\x00\x00\x01\x5D\x00\x00\x00\x03$7',b'\x00\x01\x57\x11S_un_b',b'\x00\x01\x58\x11S_un_w',b'\x00\x00\x1A\x11S_addr'),(b'\x00\x00\x01\x57\x00\x00\x00\x02$8',b'\x00\x01\x5F\x11s_b1',b'\x00\x01\x5F\x11s_b2',b'\x00\x01\x5F\x11s_b3',b'\x00\x01\x5F\x11s_b4'),(b'\x00\x00\x01\x58\x00\x00\x00\x02$9',b'\x00\x00\xDD\x11s_w1',b'\x00\x00\xDD\x11s_w2'),(b'\x00\x00\x01\x3C\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x11\x11hProcess',b'\x00\x00\x11\x11hThread',b'\x00\x00\x1A\x11dwProcessId',b'\x00\x00\x1A\x11dwThreadId'),(b'\x00\x00\x01\x40\x00\x00\x00\x00$SCOPE_ID',b'\x00\x01\x5B\x11'),(b'\x00\x00\x01\x47\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x1A\x11cb',b'\x00\x00\x27\x11lpReserved',b'\x00\x00\x27\x11lpDesktop',b'\x00\x00\x27\x11lpTitle',b'\x00\x00\x1A\x11dwX',b'\x00\x00\x1A\x11dwY',b'\x00\x00\x1A\x11dwXSize',b'\x00\x00\x1A\x11dwYSize',b'\x00\x00\x1A\x11dwXCountChars',b'\x00\x00\x1A\x11dwYCountChars',b'\x00\x00\x1A\x11dwFillAttribute',b'\x00\x00\x1A\x11dwFlags',b'\x00\x00\xDD\x11wShowWindow',b'\x00\x00\xDD\x11cbReserved2',b'\x00\x01\x5E\x11lpReserved2',b'\x00\x00\x11\x11hStdInput',b'\x00\x00\x11\x11hStdOutput',b'\x00\x00\x11\x11hStdError'),(b'\x00\x00\x01\x36\x00\x00\x00\x02_GUID',b'\x00\x00\x1A\x11Data1',b'\x00\x00\xDD\x11Data2',b'\x00\x00\xDD\x11Data3',b'\x00\x01\x62\x11Data4'),(b'\x00\x00\x01\x3B\x00\x00\x00\x02_OVERLAPPED',b'\x00\x00\x1A\x11Internal',b'\x00\x00\x1A\x11InternalHigh',b'\x00\x01\x59\x11DUMMYUNIONNAME',b'\x00\x00\x11\x11hEvent'),(b'\x00\x00\x01\x3E\x00\x00\x00\x02_PostCallbackData',b'\x00\x00\x11\x11hCompletionPort',b'\x00\x00\x22\x11Overlapped'),(b'\x00\x00\x01\x41\x00\x00\x00\x02_SECURITY_ATTRIBUTES',b'\x00\x00\x1A\x11nLength',b'\x00\x00\x11\x11lpSecurityDescriptor',b'\x00\x00\x01\x11bInheritHandle'),(b'\x00\x00\x01\x48\x00\x00\x00\x02_WSABUF',b'\x00\x00\x1A\x11len',b'\x00\x00\x27\x11buf'),(b'\x00\x00\x01\x4A\x00\x00\x00\x02_WSAPROTOCOLCHAIN',b'\x00\x00\x01\x11ChainLen',b'\x00\x01\x64\x11ChainEntries'),(b'\x00\x00\x01\x4B\x00\x00\x00\x02_WSAPROTOCOL_INFOW',b'\x00\x00\x1A\x11dwServiceFlags1',b'\x00\x00\x1A\x11dwServiceFlags2',b'\x00\x00\x1A\x11dwServiceFlags3',b'\x00\x00\x1A\x11dwServiceFlags4',b'\x00\x00\x1A\x11dwProviderFlags',b'\x00\x01\x36\x11ProviderId',b'\x00\x00\x1A\x11dwCatalogEntryId',b'\x00\x01\x4A\x11ProtocolChain',b'\x00\x00\x01\x11iVersion',b'\x00\x00\x01\x11iAddressFamily',b'\x00\x00\x01\x11iMaxSockAddr',b'\x00\x00\x01\x11iMinSockAddr',b'\x00\x00\x01\x11iSocketType',b'\x00\x00\x01\x11iProtocol',b'\x00\x00\x01\x11iProtocolMaxOffset',b'\x00\x00\x01\x11iNetworkByteOrder',b'\x00\x00\x01\x11iSecurityScheme',b'\x00\x00\x1A\x11dwMessageSize',b'\x00\x00\x1A\x11dwProviderReserved',b'\x00\x01\x69\x11szProtocol'),(b'\x00\x00\x01\x38\x00\x00\x00\x02in6_addr',b'\x00\x01\x5A\x11u'),(b'\x00\x00\x01\x3A\x00\x00\x00\x02in_addr',b'\x00\x01\x5D\x11S_un'),(b'\x00\x00\x01\x42\x00\x00\x00\x02sockaddr',b'\x00\x00\xDD\x11sa_family',b'\x00\x01\x4D\x11sa_data'),(b'\x00\x00\x01\x46\x00\x00\x00\x02sockaddr_in',b'\x00\x01\x54\x11sin_family',b'\x00\x00\xDD\x11sin_port',b'\x00\x01\x3A\x11sin_addr',b'\x00\x01\x4F\x11sin_zero'),(b'\x00\x00\x01\x45\x00\x00\x00\x00sockaddr_in6',b'\x00\x00\xDD\x11sin6_family',b'\x00\x00\xDD\x11sin6_port',b'\x00\x00\x1A\x11sin6_flowinfo',b'\x00\x01\x38\x11sin6_addr',b'\x00\x01\x5C\x11')),
-    _typenames = (b'\x00\x00\x00\xDDADDRESS_FAMILY',b'\x00\x00\x01\x53AcceptExPtr',b'\x00\x00\x01\x52ConnectExPtr',b'\x00\x00\x01\x51DisconnectExPtr',b'\x00\x00\x01\x36GUID',b'\x00\x00\x01\x38IN6_ADDR',b'\x00\x00\x01\x3AINADDR',b'\x00\x00\x01\x51LPFN_DISCONNECTEX',b'\x00\x00\x01\x37LPIN6_ADDR',b'\x00\x00\x00\x22LPOVERLAPPED',b'\x00\x00\x00\x63LPOVERLAPPED_COMPLETION_ROUTINE',b'\x00\x00\x00\x30LPPROCESS_INFORMATION',b'\x00\x00\x01\x3DLPPostCallbackData',b'\x00\x00\x00\xECLPSECURITY_ATTRIBUTES',b'\x00\x00\x00\x15LPSOCKADDR',b'\x00\x00\x01\x43LPSOCKADDR_IN',b'\x00\x00\x01\x44LPSOCKADDR_IN6_LH',b'\x00\x00\x00\x2FLPSTARTUPINFO',b'\x00\x00\x00\x5ELPWSABUF',b'\x00\x00\x01\x49LPWSAPROTOCOLCHAIN',b'\x00\x00\x00\xB5LPWSAPROTOCOL_INFOW',b'\x00\x00\x01\x3BOVERLAPPED',b'\x00\x00\x01\x37PIN6_ADDR',b'\x00\x00\x01\x39PINADDR',b'\x00\x00\x01\x3CPROCESS_INFORMATION',b'\x00\x00\x01\x3FPSCOPE_ID',b'\x00\x00\x00\xECPSECURITY_ATTRIBUTES',b'\x00\x00\x00\x15PSOCKADDR',b'\x00\x00\x01\x43PSOCKADDR_IN',b'\x00\x00\x01\x44PSOCKADDR_IN6_LH',b'\x00\x00\x01\x3EPostCallbackData',b'\x00\x00\x01\x40SCOPE_ID',b'\x00\x00\x01\x41SECURITY_ATTRIBUTES',b'\x00\x00\x01\x42SOCKADDR',b'\x00\x00\x01\x46SOCKADDR_IN',b'\x00\x00\x01\x45SOCKADDR_IN6_LH',b'\x00\x00\x00\x11SOCKET',b'\x00\x00\x01\x47STARTUPINFO',b'\x00\x00\x00\x3BWAITORTIMERCALLBACK',b'\x00\x00\x01\x48WSABUF',b'\x00\x00\x01\x4AWSAPROTOCOLCHAIN',b'\x00\x00\x01\x4BWSAPROTOCOL_INFOW',b'\x00\x00\x00\xDDwint_t'),
+    _types = b'\x00\x00\x05\x0D\x00\x01\x63\x03\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x01\x7F\x03\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x01\x59\x03\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x03\x00\x01\x52\x03\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x01\x63\x03\x00\x00\x2B\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x2B\x11\x00\x00\x2B\x11\x00\x01\x5E\x03\x00\x01\x53\x03\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x03\x00\x00\x37\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x37\x11\x00\x00\x15\x11\x00\x01\x49\x03\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x26\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x26\x11\x00\x00\x25\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x26\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x19\x11\x00\x00\x07\x01\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x26\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x01\x5F\x03\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x25\x11\x00\x00\x26\x11\x00\x01\x41\x03\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x62\x11\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x0A\x01\x00\x00\x26\x11\x00\x00\x67\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x25\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x25\x11\x00\x00\x25\x03\x00\x00\x26\x03\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x25\x11\x00\x00\x25\x11\x00\x00\x25\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x26\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x26\x11\x00\x00\x67\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x25\x11\x00\x00\x25\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x26\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x37\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x01\x7F\x03\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x26\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\xEE\x03\x00\x00\x07\x01\x00\x01\x62\x03\x00\x00\x19\x11\x00\x00\x05\x03\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\xBF\x11\x00\x00\xBF\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\xBF\x11\x00\x00\xBF\x11\x00\x00\x33\x11\x00\x00\x34\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x74\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x02\x0D\x00\x00\x0A\x01\x00\x00\x37\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x02\x0D\x00\x00\x15\x11\x00\x00\xBF\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x02\x0D\x00\x00\x02\x0F\x00\x00\xE9\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\xE9\x0D\x00\x00\x00\x0F\x00\x00\xE9\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x01\x58\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x01\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\xFD\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xEE\x03\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x01\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xFD\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x01\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xFD\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\xBF\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xFD\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x01\x06\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xFD\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x01\x7F\x0D\x00\x00\x08\x01\x00\x00\x00\x0F\x00\x01\x7F\x0D\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x00\x0F\x00\x01\x7F\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x01\x7F\x0D\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x0C\x09\x00\x01\x4F\x03\x00\x00\x13\x09\x00\x01\x51\x03\x00\x00\x14\x09\x00\x00\x0D\x09\x00\x00\x09\x09\x00\x01\x55\x03\x00\x00\x0E\x09\x00\x01\x57\x03\x00\x00\x0A\x09\x00\x00\x0F\x09\x00\x00\x15\x09\x00\x01\x5D\x03\x00\x01\x5C\x03\x00\x00\x17\x09\x00\x00\x16\x09\x00\x00\x0B\x09\x00\x00\x10\x09\x00\x01\x61\x03\x00\x00\x11\x09\x00\x00\x12\x09\x00\x00\x02\x01\x00\x01\x63\x05\x00\x00\x00\x0E\x00\x01\x63\x05\x00\x00\x00\x08\x00\x00\x51\x03\x00\x00\x57\x03\x00\x00\xA4\x03\x00\x00\x05\x01\x00\x00\x01\x09\x00\x00\x04\x09\x00\x00\x07\x09\x00\x00\x08\x09\x00\x00\x00\x09\x00\x00\x02\x09\x00\x00\x03\x09\x00\x00\x05\x09\x00\x00\x06\x09\x00\x01\x76\x03\x00\x00\x04\x01\x00\x01\x76\x05\x00\x00\x00\x10\x00\x01\x76\x05\x00\x00\x00\x08\x00\x00\x02\x05\x00\x00\x00\x07\x00\x00\xE9\x05\x00\x00\x00\x08\x00\x00\x00\x01\x00\x00\xEE\x05\x00\x00\x01\x00',
+    _globals = (b'\x00\x00\x44\x23CancelIo',0,b'\x00\x00\x47\x23CancelIoEx',0,b'\x00\x00\x44\x23CloseHandle',0,b'\x00\x00\x47\x23ConnectNamedPipe',0,b'\x00\x00\xFC\x23CreateEventA',0,b'\x00\x01\x02\x23CreateEventW',0,b'\x00\x01\x08\x23CreateFileA',0,b'\x00\x01\x35\x23CreateFileW',0,b'\x00\x01\x23\x23CreateIoCompletionPort',0,b'\x00\x01\x11\x23CreateNamedPipeA',0,b'\x00\x01\x2B\x23CreateNamedPipeW',0,b'\x00\x00\x36\x23CreatePipe',0,b'\x00\x00\x2A\x23CreateProcessA',0,b'\x00\x00\xC5\x23CreateProcessW',0,b'\x00\x00\xAE\x23DuplicateHandle',0,b'\x00\x01\x3E\x23ExitProcess',0,b'\x00\x01\x29\x23GetCurrentProcess',0,b'\x00\x00\x76\x23GetExitCodeProcess',0,b'\x00\x00\xE6\x23GetLastError',0,b'\x00\x00\xE1\x23GetModuleFileNameW',0,b'\x00\x00\x4B\x23GetOverlappedResult',0,b'\x00\x00\xFA\x23GetProcessHeap',0,b'\x00\x00\x7A\x23GetQueuedCompletionStatus',0,b'\x00\x01\x20\x23GetStdHandle',0,b'\x00\x00\xE6\x23GetVersion',0,b'\x00\x00\xF5\x23HeapAlloc',0,b'\x00\x00\x1C\x23HeapFree',0,b'\xFF\xFF\xFF\x1FMAX_PROTOCOL_CHAIN',7,b'\x00\x00\xF0\x23OpenProcess',0,b'\x00\x00\x9C\x23PeekNamedPipe',0,b'\x00\x00\x87\x23PostQueuedCompletionStatus',0,b'\x00\x00\x21\x23ReadFile',0,b'\x00\x00\x3C\x23RegisterWaitForSingleObject',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\xD4\x23SetErrorMode',0,b'\x00\x00\x44\x23SetEvent',0,b'\x00\x01\x46\x23SetLastError',0,b'\x00\x00\x81\x23SetNamedPipeHandleState',0,b'\x00\x00\x72\x23TerminateProcess',0,b'\x00\x00\x44\x23UnregisterWait',0,b'\x00\x00\x98\x23UnregisterWaitEx',0,b'\x00\x00\x8D\x23WSAIoctl',0,b'\xFF\xFF\xFF\x1FWSAPROTOCOL_LEN',255,b'\x00\x00\x60\x23WSARecv',0,b'\x00\x00\x69\x23WSASend',0,b'\x00\x00\xBE\x23WSAStringToAddressW',0,b'\xFF\xFF\xFF\x1FWT_EXECUTEINWAITTHREAD',4,b'\xFF\xFF\xFF\x1FWT_EXECUTEONLYONCE',8,b'\x00\x00\xD7\x23WaitForMultipleObjects',0,b'\x00\x00\xDD\x23WaitForSingleObject',0,b'\x00\x00\x00\x23WaitNamedPipeA',0,b'\x00\x00\xB7\x23WriteFile',0,b'\x00\x00\xD1\x23_get_osfhandle',0,b'\x00\x00\x28\x23_getch',0,b'\x00\x00\x28\x23_getche',0,b'\x00\x00\xEB\x23_getwch',0,b'\x00\x00\xEB\x23_getwche',0,b'\x00\x00\x28\x23_kbhit',0,b'\x00\x00\x0B\x23_locking',0,b'\x00\x00\x10\x23_open_osfhandle',0,b'\x00\x00\x04\x23_putch',0,b'\x00\x00\xED\x23_putwch',0,b'\x00\x00\x07\x23_setmode',0,b'\x00\x00\x04\x23_ungetch',0,b'\x00\x00\xE8\x23_ungetwch',0,b'\x00\x00\x17\x23bind',0,b'\x00\x00\x14\x23closesocket',0,b'\x00\x00\xE8\x23htons',0,b'\x00\x01\x1B\x23socket',0),
+    _struct_unions = ((b'\x00\x00\x01\x70\x00\x00\x00\x03$1',b'\x00\x01\x6C\x11DUMMYSTRUCTNAME',b'\x00\x00\x15\x11Pointer'),(b'\x00\x00\x01\x6C\x00\x00\x00\x02$2',b'\x00\x00\x02\x11Offset',b'\x00\x00\x02\x11OffsetHigh'),(b'\x00\x00\x01\x71\x00\x00\x00\x03$3',b'\x00\x01\x77\x11Byte',b'\x00\x01\x7D\x11Word'),(b'\x00\x00\x01\x72\x00\x00\x00\x01$4',b'\x00\x01\x6D\x11',b'\x00\x00\x02\x11Value'),(b'\x00\x00\x01\x6D\x00\x00\x00\x02$5',b'\x00\x00\x02\x13\x00\x00\x00\x1CZone',b'\x00\x00\x02\x13\x00\x00\x00\x04Level'),(b'\x00\x00\x01\x73\x00\x00\x00\x03$6',b'\x00\x00\x02\x11sin6_scope_id',b'\x00\x01\x57\x11sin6_scope_struct'),(b'\x00\x00\x01\x74\x00\x00\x00\x03$7',b'\x00\x01\x6E\x11S_un_b',b'\x00\x01\x6F\x11S_un_w',b'\x00\x00\x02\x11S_addr'),(b'\x00\x00\x01\x6E\x00\x00\x00\x02$8',b'\x00\x01\x76\x11s_b1',b'\x00\x01\x76\x11s_b2',b'\x00\x01\x76\x11s_b3',b'\x00\x01\x76\x11s_b4'),(b'\x00\x00\x01\x6F\x00\x00\x00\x02$9',b'\x00\x00\xE9\x11s_w1',b'\x00\x00\xE9\x11s_w2'),(b'\x00\x00\x01\x53\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x02\x11dwProcessId',b'\x00\x00\x02\x11dwThreadId'),(b'\x00\x00\x01\x57\x00\x00\x00\x00$SCOPE_ID',b'\x00\x01\x72\x11'),(b'\x00\x00\x01\x5E\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x02\x11cb',b'\x00\x00\x2B\x11lpReserved',b'\x00\x00\x2B\x11lpDesktop',b'\x00\x00\x2B\x11lpTitle',b'\x00\x00\x02\x11dwX',b'\x00\x00\x02\x11dwY',b'\x00\x00\x02\x11dwXSize',b'\x00\x00\x02\x11dwYSize',b'\x00\x00\x02\x11dwXCountChars',b'\x00\x00\x02\x11dwYCountChars',b'\x00\x00\x02\x11dwFillAttribute',b'\x00\x00\x02\x11dwFlags',b'\x00\x00\xE9\x11wShowWindow',b'\x00\x00\xE9\x11cbReserved2',b'\x00\x01\x75\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError'),(b'\x00\x00\x01\x4D\x00\x00\x00\x02_GUID',b'\x00\x00\x02\x11Data1',b'\x00\x00\xE9\x11Data2',b'\x00\x00\xE9\x11Data3',b'\x00\x01\x79\x11Data4'),(b'\x00\x00\x01\x52\x00\x00\x00\x02_OVERLAPPED',b'\x00\x00\x02\x11Internal',b'\x00\x00\x02\x11InternalHigh',b'\x00\x01\x70\x11DUMMYUNIONNAME',b'\x00\x00\x15\x11hEvent'),(b'\x00\x00\x01\x55\x00\x00\x00\x02_PostCallbackData',b'\x00\x00\x15\x11hCompletionPort',b'\x00\x00\x26\x11Overlapped'),(b'\x00\x00\x01\x58\x00\x00\x00\x02_SECURITY_ATTRIBUTES',b'\x00\x00\x02\x11nLength',b'\x00\x00\x15\x11lpSecurityDescriptor',b'\x00\x00\x05\x11bInheritHandle'),(b'\x00\x00\x01\x5F\x00\x00\x00\x02_WSABUF',b'\x00\x00\x02\x11len',b'\x00\x00\x2B\x11buf'),(b'\x00\x00\x01\x61\x00\x00\x00\x02_WSAPROTOCOLCHAIN',b'\x00\x00\x05\x11ChainLen',b'\x00\x01\x7B\x11ChainEntries'),(b'\x00\x00\x01\x62\x00\x00\x00\x02_WSAPROTOCOL_INFOW',b'\x00\x00\x02\x11dwServiceFlags1',b'\x00\x00\x02\x11dwServiceFlags2',b'\x00\x00\x02\x11dwServiceFlags3',b'\x00\x00\x02\x11dwServiceFlags4',b'\x00\x00\x02\x11dwProviderFlags',b'\x00\x01\x4D\x11ProviderId',b'\x00\x00\x02\x11dwCatalogEntryId',b'\x00\x01\x61\x11ProtocolChain',b'\x00\x00\x05\x11iVersion',b'\x00\x00\x05\x11iAddressFamily',b'\x00\x00\x05\x11iMaxSockAddr',b'\x00\x00\x05\x11iMinSockAddr',b'\x00\x00\x05\x11iSocketType',b'\x00\x00\x05\x11iProtocol',b'\x00\x00\x05\x11iProtocolMaxOffset',b'\x00\x00\x05\x11iNetworkByteOrder',b'\x00\x00\x05\x11iSecurityScheme',b'\x00\x00\x02\x11dwMessageSize',b'\x00\x00\x02\x11dwProviderReserved',b'\x00\x01\x80\x11szProtocol'),(b'\x00\x00\x01\x4F\x00\x00\x00\x02in6_addr',b'\x00\x01\x71\x11u'),(b'\x00\x00\x01\x51\x00\x00\x00\x02in_addr',b'\x00\x01\x74\x11S_un'),(b'\x00\x00\x01\x59\x00\x00\x00\x02sockaddr',b'\x00\x00\xE9\x11sa_family',b'\x00\x01\x64\x11sa_data'),(b'\x00\x00\x01\x5D\x00\x00\x00\x02sockaddr_in',b'\x00\x01\x6B\x11sin_family',b'\x00\x00\xE9\x11sin_port',b'\x00\x01\x51\x11sin_addr',b'\x00\x01\x66\x11sin_zero'),(b'\x00\x00\x01\x5C\x00\x00\x00\x00sockaddr_in6',b'\x00\x00\xE9\x11sin6_family',b'\x00\x00\xE9\x11sin6_port',b'\x00\x00\x02\x11sin6_flowinfo',b'\x00\x01\x4F\x11sin6_addr',b'\x00\x01\x73\x11')),
+    _typenames = (b'\x00\x00\x00\xE9ADDRESS_FAMILY',b'\x00\x00\x01\x6AAcceptExPtr',b'\x00\x00\x01\x69ConnectExPtr',b'\x00\x00\x01\x68DisconnectExPtr',b'\x00\x00\x01\x4DGUID',b'\x00\x00\x01\x4FIN6_ADDR',b'\x00\x00\x01\x51INADDR',b'\x00\x00\x01\x68LPFN_DISCONNECTEX',b'\x00\x00\x01\x4ELPIN6_ADDR',b'\x00\x00\x00\x26LPOVERLAPPED',b'\x00\x00\x00\x67LPOVERLAPPED_COMPLETION_ROUTINE',b'\x00\x00\x00\x34LPPROCESS_INFORMATION',b'\x00\x00\x01\x54LPPostCallbackData',b'\x00\x00\x00\xFDLPSECURITY_ATTRIBUTES',b'\x00\x00\x00\x19LPSOCKADDR',b'\x00\x00\x01\x5ALPSOCKADDR_IN',b'\x00\x00\x01\x5BLPSOCKADDR_IN6_LH',b'\x00\x00\x00\x33LPSTARTUPINFO',b'\x00\x00\x00\x62LPWSABUF',b'\x00\x00\x01\x60LPWSAPROTOCOLCHAIN',b'\x00\x00\x00\xC1LPWSAPROTOCOL_INFOW',b'\x00\x00\x01\x52OVERLAPPED',b'\x00\x00\x01\x4EPIN6_ADDR',b'\x00\x00\x01\x50PINADDR',b'\x00\x00\x01\x53PROCESS_INFORMATION',b'\x00\x00\x01\x56PSCOPE_ID',b'\x00\x00\x00\xFDPSECURITY_ATTRIBUTES',b'\x00\x00\x00\x19PSOCKADDR',b'\x00\x00\x01\x5APSOCKADDR_IN',b'\x00\x00\x01\x5BPSOCKADDR_IN6_LH',b'\x00\x00\x01\x55PostCallbackData',b'\x00\x00\x01\x57SCOPE_ID',b'\x00\x00\x01\x58SECURITY_ATTRIBUTES',b'\x00\x00\x01\x59SOCKADDR',b'\x00\x00\x01\x5DSOCKADDR_IN',b'\x00\x00\x01\x5CSOCKADDR_IN6_LH',b'\x00\x00\x00\x15SOCKET',b'\x00\x00\x01\x5ESTARTUPINFO',b'\x00\x00\x00\x3FWAITORTIMERCALLBACK',b'\x00\x00\x01\x5FWSABUF',b'\x00\x00\x01\x61WSAPROTOCOLCHAIN',b'\x00\x00\x01\x62WSAPROTOCOL_INFOW',b'\x00\x00\x00\xE9wint_t'),
 )
diff --git a/lib_pypy/_winapi.py b/lib_pypy/_winapi.py
--- a/lib_pypy/_winapi.py
+++ b/lib_pypy/_winapi.py
@@ -5,6 +5,7 @@
 """
 
 import sys
+
 if sys.platform != 'win32':
     raise ImportError("The '_winapi' module is only available on Windows")
 
@@ -16,6 +17,12 @@
 GetVersion = _kernel32.GetVersion
 NULL = _ffi.NULL
 
+def SetLastError(errno):
+    return _kernel32.SetLastError(errno)
+
+def GetLastError():
+    return _kernel32.GetLastError()
+
 # Now the _subprocess module implementation
 def _WinError(type=WindowsError):
     code, message = _ffi.getwinerror()
@@ -51,19 +58,19 @@
     res = _kernel32.CreatePipe(handles, handles + 1, NULL, size)
 
     if not res:
-        SetFromWindowsErr(0)
+        SetFromWindowsErr(GetLastError())
 
     return _handle2int(handles[0]), _handle2int(handles[1])
 
 def CreateNamedPipe(*args):
     handle = _kernel32.CreateNamedPipeW(*args)
-    if handle == INVALID_HANDLE_VALUE:
+    if handle == _INVALID_HANDLE_VALUE:
         SetFromWindowsErr(0)
     return _handle2int(handle)
 
 def CreateFile(*args):
     handle = _kernel32.CreateFileW(*args)
-    if handle == INVALID_HANDLE_VALUE:
+    if handle == _INVALID_HANDLE_VALUE:
         SetFromWindowsErr(0)
     return _handle2int(handle)
 
@@ -77,14 +84,14 @@
         d2 = NULL
     else:
         d2 = _ffi.new('DWORD[1]', [collect_data_timeout])
-    ret = _kernel32.SetNamedPipeHandleState(namedpipe, d0, d1, d2)
+    ret = _kernel32.SetNamedPipeHandleState(_int2handle(namedpipe), d0, d1, d2)
     if not ret:
         raise _WinError()
 
 class Overlapped(object):
     def __init__(self, handle):
         self.overlapped = _ffi.new('OVERLAPPED[1]')
-        self.handle = handle
+        self.handle = _handle2int(handle)
         self.readbuffer = None
         self.pending = 0
         self.completed = 0
@@ -96,47 +103,140 @@
         # do this somehow else
         err = _kernel32.GetLastError()
         bytes = _ffi.new('DWORD[1]')
-        o = self.overlapped[0]
         if self.pending:
-            if _kernel32.CancelIoEx(o.handle, o.overlapped) & \
-                self.GetOverlappedResult(o.handle, o.overlapped, _ffi.addressof(bytes), True):
+            result = _kernel32.CancelIoEx(_int2handle(self.handle), self.overlapped)
+            if result: 
+                _kernel32.GetOverlappedResult(_int2handle(self.handle), self.overlapped, bytes, True)
                 # The operation is no longer pending, nothing to do
-                pass
-            else:
-                raise RuntimeError('deleting an overlapped struct with a pending operation not supported')
+                
+            #else:
+                # We need to raise a warning here and not crash pypy
+                #raise RuntimeError('deleting an overlapped struct with a pending operation not supported')
+        CloseHandle(_int2handle(self.overlapped[0].hEvent))
+        _kernel32.SetLastError(err)
 
     @property
     def event(self):
-        xxx
-        return None
+        return _handle2int(self.overlapped[0].hEvent)
 
     def GetOverlappedResult(self, wait):
         transferred = _ffi.new('DWORD[1]', [0])
-        res = _kernel32.GetOverlappedResult(self.handle, self.overlapped, transferred, wait != 0)
+        res = _kernel32.GetOverlappedResult(_int2handle(self.handle), self.overlapped, transferred, wait != 0)
+        
         if res:
             err = ERROR_SUCCESS
         else:
-            err = GetLastError()
+            err = _kernel32.GetLastError()
+
         if err in (ERROR_SUCCESS, ERROR_MORE_DATA, ERROR_OPERATION_ABORTED):
             self.completed = 1
             self.pending = 0
-        elif res == ERROR_IO_INCOMPLETE:
-            pass
-        else:
+        elif res != ERROR_IO_INCOMPLETE:
             self.pending = 0
-            raise _WinError()
+            raise _WinError(IOError)
+
         if self.completed and self.readbuffer:
-            if transferred != len(self.readbuffer):
-                raise _WinError()
+            if transferred[0] != len(self.readbuffer):
+                tempbuffer = _ffi.new("CHAR[]", transferred[0])
+                _ffi.memmove(tempbuffer, self.readbuffer, transferred[0])
+                self.readbuffer = tempbuffer
         return transferred[0], err
 
     def getbuffer(self):
-        xxx
+        if not self.completed:
+            raise ValueError("can't get read buffer before GetOverlappedResult() "
+                        "signals the operation completed")
+        if self.readbuffer:
+            result = _ffi.buffer(self.readbuffer)
+        else:
+            result = None
+        return result
+
+    def cancel(self):
+        ret = True
+        if self.pending:
+            ret = _kernel32.CancelIoEx(_int2handle(self.handle), self.overlapped)
+        if not ret and _kernel32.GetLastError() != ERROR_NOT_FOUND:
+            return _WinError(IOError)
+        self.pending = 0
         return None
 
-    def cancel(self):
-        xxx
-        return None
+
+def ReadFile(handle, size, overlapped):
+    nread = _ffi.new("DWORD*")
+    err = _ffi.new("DWORD*")
+    use_overlapped = overlapped
+    overlapped = None
+
+    buf = _ffi.new("CHAR[]", size)
+    if not buf:
+        return _ffi.NULL
+
+    if use_overlapped:
+        overlapped = Overlapped(handle)
+        if not overlapped:
+            return _ffi.NULL
+        overlapped.readbuffer = buf
+    
+    if overlapped:
+        ret = _kernel32.ReadFile(_int2handle(handle), buf, size, nread,
+                       overlapped.overlapped)
+    else:
+        ret = _kernel32.ReadFile(_int2handle(handle), buf, size, nread,
+                       _ffi.NULL)
+ 
+    if ret:
+        err = 0
+    else: 
+        err = _kernel32.GetLastError()
+
+    if use_overlapped:
+        if not ret:
+            if err == ERROR_IO_PENDING:
+                overlapped.pending = 1
+            elif err != ERROR_MORE_DATA:
+                return _WinError(IOError)
+        return overlapped, err
+
+    if (not ret) and err != ERROR_MORE_DATA:
+        return _WinError(IOError)
+    return buf, err
+
+def WriteFile(handle, buffer, overlapped=False):
+    written = _ffi.new("DWORD*")
+    err = _ffi.new("DWORD*")
+    use_overlapped = overlapped
+    overlapped = None    
+    if use_overlapped:
+        overlapped = Overlapped(handle)
+        if not overlapped:
+            return _ffi.NULL
+        overlapped.writebuffer = bytes(buffer)
+        buf = overlapped.writebuffer
+    else:
+        buf = _ffi.new("CHAR[]", bytes(buffer))
+    if use_overlapped:
+        ret = _kernel32.WriteFile(_int2handle(handle), buf , len(buf), written, overlapped.overlapped)
+    else:
+        ret = _kernel32.WriteFile(_int2handle(handle), buf , len(buf), written, _ffi.NULL)
+
+    if ret:
+        err = 0
+    else: 
+        err = _kernel32.GetLastError()
+
+    if use_overlapped:
+        if not ret:
+            if err == ERROR_IO_PENDING:
+                overlapped.pending = 1
+            else:
+                return _WinError(IOError)
+        return overlapped, err
+
+    if not ret:
+        return _WinError(IOError)
+     
+    return written[0], err
 
  
 def ConnectNamedPipe(handle, overlapped=False):
@@ -176,7 +276,7 @@
 
     if not res:
         raise _WinError()
-
+    
     return _handle2int(target[0])
 
 def _Z(input):
@@ -225,18 +325,65 @@
             pi.dwProcessId,
             pi.dwThreadId)
 
+def OpenProcess(desired_access, inherit_handle, process_id):
+    handle = _kernel32.OpenProcess(desired_access, inherit_handle, process_id)
+    if handle == _ffi.NULL:
+        SetFromWindowsErr(0)
+        handle = INVALID_HANDLE_VALUE
+
+    return _handle2int(handle)
+
+def PeekNamedPipe(handle, size=0):
+    nread = _ffi.new("DWORD*")
+    navail = _ffi.new("DWORD*")
+    nleft = _ffi.new("DWORD*")
+
+    if size < 0:
+        raise ValueError("negative size")
+
+    if size:
+        buf = _ffi.new("CHAR[]", size)
+        if not buf:
+            return _ffi.NULL
+
+        ret = _kernel32.PeekNamedPipe(_int2handle(handle), buf, size, nread,
+                                      navail, nleft)
+        if not ret:
+            # In CPython SetExcFromWindowsErr is called here.
+            # Not sure what that is doing currently.
+            SetFromWindowsErr(0)
+
+
+        return  buf, navail[0], nleft[0]
+    else:
+        ret = _kernel32.PeekNamedPipe(_int2handle(handle), _ffi.NULL, 0, _ffi.NULL, navail, nleft)
+        if not ret:
+            # In CPython SetExcFromWindowsErr is called here.
+            # Not sure what that is doing currently.
+            SetFromWindowsErr(0)
+        return  navail[0], nleft[0]
+
 def WaitForSingleObject(handle, milliseconds):
     # CPython: the first argument is expected to be an integer.
     res = _kernel32.WaitForSingleObject(_int2handle(handle), milliseconds)
     if res < 0:
         raise _WinError()
+    return res
 
-    return res
+
+def WaitNamedPipe(namedpipe, milliseconds):
+    namedpipe = _ffi.new("CHAR[]", namedpipe.encode("ascii", "ignore"))
+    res = _kernel32.WaitNamedPipeA(namedpipe, milliseconds)
+
+    if res < 0:
+        raise SetFromWindowsErr(0)
+
 
 def WaitForMultipleObjects(handle_sequence, waitflag, milliseconds):
     if len(handle_sequence) > MAXIMUM_WAIT_OBJECTS:
         return None
-    
+    handle_sequence = list(map(_int2handle, handle_sequence))
+    handle_sequence = _ffi.new("HANDLE[]", handle_sequence)
     # CPython makes the wait interruptible by ctrl-c. We need to add this in at some point
     res = _kernel32.WaitForMultipleObjects(len(handle_sequence), handle_sequence, waitflag, milliseconds)
 
@@ -288,6 +435,9 @@
         raise _WinError()
     return _ffi.string(buf)
 
+def ExitProcess(exitcode):
+    _kernel32.ExitProcess(exitcode)
+
 ZERO_MEMORY = 0x00000008
 
 def malloc(size):
@@ -301,6 +451,7 @@
 STD_OUTPUT_HANDLE = -11
 STD_ERROR_HANDLE = -12
 DUPLICATE_SAME_ACCESS = 2
+DUPLICATE_CLOSE_SOURCE = 1
 STARTF_USESTDHANDLES = 0x100
 STARTF_USESHOWWINDOW = 0x001
 SW_HIDE = 0
@@ -318,15 +469,19 @@
 ERROR_SUCCESS           = 0
 ERROR_NETNAME_DELETED   = 64
 ERROR_BROKEN_PIPE       = 109
-ERROR_PIPE_BUSY         = 231 
+ERROR_SEM_TIMEOUT       = 121
+ERROR_PIPE_BUSY         = 231
+ERROR_NO_DATA           = 232 
 ERROR_MORE_DATA         = 234
 ERROR_PIPE_CONNECTED    = 535
 ERROR_OPERATION_ABORTED = 995
 ERROR_IO_INCOMPLETE     = 996
 ERROR_IO_PENDING        = 997
+ERROR_NOT_FOUND          = 1168
 ERROR_CONNECTION_REFUSED = 1225
 ERROR_CONNECTION_ABORTED = 1236
 
+
 PIPE_ACCESS_INBOUND = 0x00000001
 PIPE_ACCESS_OUTBOUND = 0x00000002
 PIPE_ACCESS_DUPLEX   = 0x00000003
@@ -362,6 +517,26 @@
 NMPWAIT_NOWAIT                =  0x00000001
 NMPWAIT_USE_DEFAULT_WAIT      =  0x00000000
 
+FILE_READ_DATA = 1
+FILE_WRITE_DATA = 2
+FILE_APPEND_DATA = 4
+FILE_READ_EA = 8
+FILE_WRITE_EA = 16
+FILE_EXECUTE = 32
+FILE_READ_ATTRIBUTES = 128 
+FILE_WRITE_ATTRIBUTES = 256
+READ_CONTROL = 0x00020000
+SYNCHRONIZE = 0x00100000
+STANDARD_RIGHTS_EXECUTE = READ_CONTROL
+STANDARD_RIGHTS_READ = READ_CONTROL
+STANDARD_RIGHTS_WRITE = READ_CONTROL
+
+FILE_GENERIC_EXECUTE = FILE_EXECUTE | FILE_READ_ATTRIBUTES | STANDARD_RIGHTS_EXECUTE | SYNCHRONIZE
+FILE_GENERIC_READ = FILE_READ_ATTRIBUTES | FILE_READ_DATA | FILE_READ_EA | STANDARD_RIGHTS_READ | SYNCHRONIZE
+FILE_GENERIC_WRITE = FILE_APPEND_DATA | FILE_WRITE_ATTRIBUTES | FILE_WRITE_DATA | FILE_WRITE_EA | STANDARD_RIGHTS_WRITE | SYNCHRONIZE
+
+PROCESS_DUP_HANDLE = 0x0040
+
 CREATE_NEW        = 1
 CREATE_ALWAYS     = 2
 OPEN_EXISTING     = 3
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -34,3 +34,7 @@
 
 Move _ssl and _hashlib from rpython to a cffi-based module, like on python3.
 Reduces the number of problematic linked-in libraries (libssl, libcrypto)
+
+.. branch: fix-vmprof-memory-tracking
+
+Fix a bug that prevent memory-tracking in vmprof working on PyPy.
diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -8,3 +8,7 @@
 .. branch: fix-literal-prev_digit-underscore
 
 Fix parsing for converting strings with underscore into ints
+
+.. branch: winmultiprocessing
+
+Improve multiprocessing support on win32
diff --git a/pypy/module/_multiprocessing/__init__.py b/pypy/module/_multiprocessing/__init__.py
--- a/pypy/module/_multiprocessing/__init__.py
+++ b/pypy/module/_multiprocessing/__init__.py
@@ -15,5 +15,5 @@
         interpleveldefs['closesocket'] = 'interp_win32_py3.multiprocessing_closesocket'
         interpleveldefs['recv'] = 'interp_win32_py3.multiprocessing_recv'
         interpleveldefs['send'] = 'interp_win32_py3.multiprocessing_send'
-    else:
-        interpleveldefs['sem_unlink'] = 'interp_semaphore.semaphore_unlink'
+    
+    interpleveldefs['sem_unlink'] = 'interp_semaphore.semaphore_unlink'
diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -36,8 +36,7 @@
         save_err=rffi.RFFI_SAVE_LASTERROR)
 
     def sem_unlink(name):
-        pass
-
+        return None
 else:
     from rpython.rlib import rposix
 
@@ -217,12 +216,13 @@
     def handle_w(space, w_handle):
         return rffi.cast(SEM_T, space.int_w(w_handle))
 
-    def semaphore_unlink(space, w_name):
-        name = space.text_w(w_name)
-        try:
-            sem_unlink(name)
-        except OSError as e:
-            raise wrap_oserror(space, e)
+# utilized by POSIX and win32
+def semaphore_unlink(space, w_name):
+    name = space.text_w(w_name)
+    try:
+        sem_unlink(name)
+    except OSError as e:
+        raise wrap_oserror(space, e)
 
 class CounterState:
     def __init__(self, space):
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -16,6 +16,7 @@
     GetSetProperty, TypeDef, generic_new_descr, interp_attrproperty,
     make_weakref_descr)
 
+_WIN32 = sys.platform.startswith('win')
 
 # XXX Hack to separate rpython and pypy
 def addr_as_object(addr, fd, space):
@@ -197,13 +198,29 @@
             self.register_finalizer(space)
 
     @unwrap_spec(family=int, type=int, proto=int,
-                 w_fileno=WrappedDefault(None))
+                 w_fdobj=WrappedDefault(None))
     def descr_init(self, space, family=AF_INET, type=SOCK_STREAM, proto=0,
-                   w_fileno=None):
+                   w_fdobj=None):
         try:
-            if not space.is_w(w_fileno, space.w_None):
-                sock = RSocket(family, type, proto,
-                               fd=space.c_filedescriptor_w(w_fileno))
+            if not space.is_w(w_fdobj, space.w_None):
+                if _WIN32 and space.isinstance_w(w_fdobj, space.w_bytes):
+                    from rpython.rlib.rsocket import _c
+                    # it is possible to pass some bytes representing a socket
+                    # in the file descriptor object on winodws
+                    fdobj = space.bytes_w(w_fdobj)
+                    info_charptr = rffi.str2charp(fdobj)
+                    try:
+                        info_ptr = rffi.cast(lltype.Ptr(_c.WSAPROTOCOL_INFOW), info_charptr)
+                        fd = _c.WSASocketW(_c.FROM_PROTOCOL_INFO, _c.FROM_PROTOCOL_INFO,
+                        _c.FROM_PROTOCOL_INFO, info_ptr, 0, _c.WSA_FLAG_OVERLAPPED)
+                        if fd == rsocket.INVALID_SOCKET:
+                            raise converted_error(space, rsocket.last_error())
+                        sock = RSocket(info_ptr.c_iAddressFamily, info_ptr.c_iSocketType, info_ptr.c_iProtocol, fd)
+                    finally:
+                        lltype.free(info_charptr, flavor='raw')
+                else:
+                    sock = RSocket(family, type, proto,
+                                   fd=space.c_filedescriptor_w(w_fdobj))
             else:
                 sock = RSocket(family, type, proto, inheritable=False)
             W_Socket.__init__(self, space, sock)
@@ -757,6 +774,26 @@
         finally:
             lltype.free(recv_ptr, flavor='raw')
 
+    @unwrap_spec(processid=int)
+    def share_w(self, space, processid):
+        from rpython.rtyper.lltypesystem import rffi, lltype
+        from rpython.rlib import rwin32
+        from rpython.rlib.rsocket import _c
+        info_ptr = lltype.malloc(_c.WSAPROTOCOL_INFOW, flavor='raw')
+        try:
+            winprocessid = rffi.cast(rwin32.DWORD, processid)
+            res = _c.WSADuplicateSocketW(
+                        self.sock.fd, winprocessid, info_ptr)
+
+            if res < 0:
+                raise converted_error(space, rsocket.last_error())
+
+            bytes_ptr = rffi.cast(rffi.CCHARP, info_ptr)
+            w_bytes = space.newbytes(rffi.charpsize2str(bytes_ptr, rffi.sizeof(_c.WSAPROTOCOL_INFOW)))
+        finally:
+            lltype.free(info_ptr, flavor='raw')
+        return w_bytes
+			
     @unwrap_spec(how="c_int")
     def shutdown_w(self, space, how):
         """shutdown(flag)
@@ -890,6 +927,7 @@
 """.split()
 if hasattr(rsocket._c, 'WSAIoctl'):
     socketmethodnames.append('ioctl')
+    socketmethodnames.append('share')
 if rsocket._c.HAVE_SENDMSG:
     socketmethodnames.append('sendmsg')
     socketmethodnames.append('recvmsg')
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -575,6 +575,27 @@
         raises(ValueError, s.ioctl, -1, None)
         s.ioctl(_socket.SIO_KEEPALIVE_VALS, (1, 100, 100))
 
+    def test_socket_sharelocal(self):
+        import _socket, sys, os
+        if sys.platform != 'win32':
+            skip("win32 only")
+        assert hasattr(_socket.socket, 'share')
+        s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM)
+        s.listen()
+        data = s.share(os.getpid())
+        # emulate socket.fromshare
+        s2 = _socket.socket(0, 0, 0, data)
+        try:
+            assert s.gettimeout() == s2.gettimeout()
+            assert s.family == s2.family
+            assert s.type == s2.type
+            if s.proto != 0:
+                assert s.proto == s2.proto
+        finally:
+            s.close()
+            s2.close()
+
+
     def test_dup(self):
         import _socket as socket, os
         s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -60,11 +60,6 @@
     'interval' is a float representing the sampling interval, in seconds.
     Must be smaller than 1.0
     """
-    w_modules = space.sys.get('modules')
-    #if space.contains_w(w_modules, space.newtext('_continuation')):
-    #    space.warn(space.newtext("Using _continuation/greenlet/stacklet together "
-    #                             "with vmprof will crash"),
-    #               space.w_RuntimeWarning)
     try:
         rvmprof.enable(fileno, period, memory, native, real_time)
     except rvmprof.VMProfError as e:
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -3989,55 +3989,6 @@
         # here it works again
         self.check_operations_history(guard_class=0, record_exact_class=1)
 
-    def test_record_exact_class_nonconst(self):
-        class Base(object):
-            def f(self):
-                raise NotImplementedError
-            def g(self):
-                raise NotImplementedError
-        class A(Base):
-            def f(self):
-                return self.a
-            def g(self):
-                return self.a + 1
-        class B(Base):
-            def f(self):
-                return self.b
-            def g(self):
-                return self.b + 1
-        class C(B):
-            def f(self):
-                self.c += 1
-                return self.c
-            def g(self):
-                return self.c + 1
-        @dont_look_inside
-        def make(x):
-            if x > 0:
-                a = A()
-                a.a = x + 1
-            elif x < 0:
-                a = B()
-                a.b = -x
-            else:
-                a = C()
-                a.c = 10
-            return a, type(a)
-        def f(x):
-            a, cls = make(x)
-            record_exact_class(a, cls)
-            if x > 0:
-                z = a.f()
-            elif x < 0:
-                z = a.f()
-            else:
-                z = a.f()
-            return z + a.g()
-        res1 = f(6)
-        res2 = self.interp_operations(f, [6])
-        assert res1 == res2
-        self.check_operations_history(guard_class=1, record_exact_class=0)
-
     def test_generator(self):
         def g(n):
             yield n+1
@@ -4889,3 +4840,52 @@
 
         res = self.meta_interp(f, [0])
         assert res == f(0)
+
+    def test_record_exact_class_nonconst(self):
+        class Base(object):
+            def f(self):
+                raise NotImplementedError
+            def g(self):
+                raise NotImplementedError
+        class A(Base):
+            def f(self):
+                return self.a
+            def g(self):
+                return self.a + 1
+        class B(Base):
+            def f(self):
+                return self.b
+            def g(self):
+                return self.b + 1
+        class C(B):
+            def f(self):
+                self.c += 1
+                return self.c
+            def g(self):
+                return self.c + 1
+        @dont_look_inside
+        def make(x):
+            if x > 0:
+                a = A()
+                a.a = x + 1
+            elif x < 0:
+                a = B()
+                a.b = -x
+            else:
+                a = C()
+                a.c = 10
+            return a, type(a)
+        def f(x):
+            a, cls = make(x)
+            record_exact_class(a, cls)
+            if x > 0:
+                z = a.f()
+            elif x < 0:
+                z = a.f()
+            else:
+                z = a.f()
+            return z + a.g()
+        res1 = f(6)
+        res2 = self.interp_operations(f, [6])
+        assert res1 == res2
+        self.check_operations_history(guard_class=1, record_exact_class=0)
diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py
--- a/rpython/rlib/_rsocket_rffi.py
+++ b/rpython/rlib/_rsocket_rffi.py
@@ -203,6 +203,7 @@
 FD_CONNECT_BIT FD_CLOSE_BIT
 WSA_IO_PENDING WSA_IO_INCOMPLETE WSA_INVALID_HANDLE
 WSA_INVALID_PARAMETER WSA_NOT_ENOUGH_MEMORY WSA_OPERATION_ABORTED
+WSA_FLAG_OVERLAPPED
 SIO_RCVALL SIO_KEEPALIVE_VALS
 
 SIOCGIFNAME SIOCGIFINDEX
@@ -225,6 +226,7 @@
                         ('SOL_UDP', 17),
                         ('SOMAXCONN', 5),
                         ('IPPROTO_IP', 6),
+                        ('IPPROTO_IPV6', 41),
                         ('IPPROTO_ICMP', 1),
                         ('IPPROTO_TCP', 6),
                         ('IPPROTO_UDP', 17),
@@ -1009,6 +1011,7 @@
     CConfig.WSAPROTOCOL_INFO = platform.Struct(
         'WSAPROTOCOL_INFOA',
         [])  # Struct is just passed between functions
+	   
     CConfig.FROM_PROTOCOL_INFO = platform.DefinedConstantInteger(
         'FROM_PROTOCOL_INFO')
 
@@ -1033,6 +1036,45 @@
         [('onoff', rffi.ULONG),
          ('keepalivetime', rffi.ULONG),
          ('keepaliveinterval', rffi.ULONG)])
+	
+    CConfig.GUID = platform.Struct(
+		     'struct _GUID',
+			 [('Data1', rffi.UINT),
+             ('Data2', rffi.UINT),
+             ('Data3', rffi.UINT),
+             ('Data4', rffi.CFixedArray(rffi.UCHAR, 8))
+         ])
+		 
+    CConfig.WSAPROTOCOLCHAIN = platform.Struct(
+        'struct _WSAPROTOCOLCHAIN',
+        [('ChainLen', rffi.INT),
+         ('ChainEntries', rffi.CFixedArray(rffi.UINT, 7))])
+	
+    WSAPROTOCOLCHAIN = CConfig.WSAPROTOCOLCHAIN
+    GUID = CConfig.GUID 
+	
+    CConfig.WSAPROTOCOL_INFOW = platform.Struct(
+        'struct _WSAPROTOCOL_INFOW',
+        [('dwServiceFlags1', rffi.UINT),
+         ('dwServiceFlags2', rffi.UINT),
+         ('dwServiceFlags3', rffi.UINT),
+         ('dwServiceFlags4', rffi.UINT),
+         ('dwProviderFlags', rffi.UINT),
+         ('ProviderId', GUID),
+         ('dwCatalogEntryId', rffi.UINT),
+         ('ProtocolChain', WSAPROTOCOLCHAIN),
+         ('iVersion', rffi.INT),
+         ('iAddressFamily', rffi.INT),
+         ('iMaxSockAddr', rffi.INT),
+         ('iMinSockAddr', rffi.INT),
+         ('iSocketType', rffi.INT),
+         ('iProtocol', rffi.INT),
+         ('iProtocolMaxOffset', rffi.INT),
+         ('iNetworkByteOrder', rffi.INT),
+         ('iSecurityScheme', rffi.INT),
+         ('dwMessageSize', rffi.UINT),
+         ('dwProviderReserved', rffi.UINT),
+         ('szProtocol',  rffi.CFixedArray(rffi.UCHAR, 256))])
 
 
 class cConfig:
@@ -1336,6 +1378,20 @@
                          rffi.VOIDP, rwin32.DWORD,
                          rwin32.LPDWORD, rffi.VOIDP, rffi.VOIDP],
                         rffi.INT, save_err=SAVE_ERR)
+
+    WSAPROTOCOL_INFOW = cConfig.WSAPROTOCOL_INFOW
+
+    WSADuplicateSocketW = external('WSADuplicateSocketW',
+                                 [socketfd_type, rwin32.DWORD,
+                                  lltype.Ptr(WSAPROTOCOL_INFOW)],
+                                  rffi.INT, save_err=SAVE_ERR)
+
+    WSASocketW = external('WSASocketW',
+                         [rffi.INT, rffi.INT, rffi.INT,
+                          lltype.Ptr(WSAPROTOCOL_INFOW),
+                          rwin32.DWORD, rwin32.DWORD],
+                         socketfd_type, save_err=SAVE_ERR)
+								  
     tcp_keepalive = cConfig.tcp_keepalive
 
     WSAPROTOCOL_INFO = cConfig.WSAPROTOCOL_INFO
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -143,7 +143,7 @@
             native = 0 # force disabled on Windows
         lines = 0 # not supported on PyPy currently
 
-        p_error = self.cintf.vmprof_init(fileno, interval, lines, memory, "pypy", native, real_time)
+        p_error = self.cintf.vmprof_init(fileno, interval, memory, lines, "pypy", native, real_time)
         if p_error:
             raise VMProfError(rffi.charp2str(p_error))
 
diff --git a/rpython/rlib/rvmprof/test/test_rvmprof.py b/rpython/rlib/rvmprof/test/test_rvmprof.py
--- a/rpython/rlib/rvmprof/test/test_rvmprof.py
+++ b/rpython/rlib/rvmprof/test/test_rvmprof.py
@@ -98,12 +98,12 @@
         self.tmpfilename = str(self.tmpfile)
         super(RVMProfSamplingTest, self).init()
 
-    ENTRY_POINT_ARGS = (int, float)
-    def entry_point(self, value, delta_t):
+    ENTRY_POINT_ARGS = (int, float, int)
+    def entry_point(self, value, delta_t, memory=0):
         code = self.MyCode('py:code:52:test_enable')
         rvmprof.register_code(code, self.MyCode.get_name)
         fd = os.open(self.tmpfilename, os.O_WRONLY | os.O_CREAT, 0666)
-        rvmprof.enable(fd, self.SAMPLING_INTERVAL)
+        rvmprof.enable(fd, self.SAMPLING_INTERVAL, memory=memory)
         start = time.time()
         res = 0
         while time.time() < start+delta_t:
@@ -128,17 +128,25 @@
 
     def test(self):
         from vmprof import read_profile
-        assert self.entry_point(10**4, 0.1) == 99990000
+        assert self.entry_point(10**4, 0.1, 0) == 99990000
         assert self.tmpfile.check()
         self.tmpfile.remove()
         #
-        assert self.rpy_entry_point(10**4, 0.5) == 99990000
+        assert self.rpy_entry_point(10**4, 0.5, 0) == 99990000
         assert self.tmpfile.check()
         prof = read_profile(self.tmpfilename)
         tree = prof.get_tree()
         assert tree.name == 'py:code:52:test_enable'
         assert self.approx_equal(tree.count, 0.5/self.SAMPLING_INTERVAL)
 
+    def test_mem(self):
+        from vmprof import read_profile
+        assert self.rpy_entry_point(10**4, 0.5, 1) == 99990000
+        assert self.tmpfile.check()
+        prof = read_profile(self.tmpfilename)
+        assert prof.profile_memory
+        assert all(p[-1] > 0 for p in prof.profiles)
+
 
 class TestNative(RVMProfSamplingTest):
 
@@ -177,7 +185,7 @@
     def test(self):
         from vmprof import read_profile
         # from vmprof.show import PrettyPrinter
-        assert self.rpy_entry_point(3, 0.5) == 42000
+        assert self.rpy_entry_point(3, 0.5, 0) == 42000
         assert self.tmpfile.check()
 
         prof = read_profile(self.tmpfilename)

From pypy.commits at gmail.com  Mon May 27 18:35:53 2019
From: pypy.commits at gmail.com (andrewjlawrence)
Date: Mon, 27 May 2019 15:35:53 -0700 (PDT)
Subject: [pypy-commit] pypy winconsoleio: Initial implementation of
 winconsoleio
Message-ID: <5cec6649.1c69fb81.e97f.e452@mx.google.com>

Author: andrewjlawrence
Branch: winconsoleio
Changeset: r96706:c795c20ce4b8
Date: 2019-05-27 23:33 +0100
http://bitbucket.org/pypy/pypy/changeset/c795c20ce4b8/

Log:	Initial implementation of winconsoleio

diff --git a/pypy/module/_io/interp_win32consoleio.py b/pypy/module/_io/interp_win32consoleio.py
--- a/pypy/module/_io/interp_win32consoleio.py
+++ b/pypy/module/_io/interp_win32consoleio.py
@@ -1,6 +1,10 @@
 import sys
 
 from pypy.module._io.interp_iobase import W_RawIOBase
+from rpython.rlib import rwin32
+
+def _pyio_get_console_type():
+    pass
 
 class W_WinConsoleIO(W_RawIOBase):
     def __init__(self, space):
@@ -9,11 +13,54 @@
     def descr_init(self, space, w_nameobj, w_mode="r", w_closefd=True, w_opener=None):
         #self.fd = -1
         #self.created = 0
-        #self.readable = 0
-        #self.writable = 0
+        self.readable = False
+        self.writable = False
         #self.closehandle = 0;
         #self.blksize = 0
-        fd = space.int_w(w_nameobj)
+        rwa = False
+        console_type = '\0'
+        self.fd = space.int_w(w_nameobj)
         if self.fd < 0:
-            raise oefmt(space.w_ValueError, "negative file descriptor")
-        self.fd = fd
+            decodedname = space.fsdecode_w(w_nameobj)
+            name = rffi.cast(rffi.CWCHARP, decodedname)
+            console_type = _pyio_get_console_type(decodedname)
+            if not console_type:
+                raise oefmt(space.w_ValueError,
+                        "Invalid console type")
+            if console_type == '\0':
+                raise oefmt(space.w_ValueError,
+                        "Cannot open non-console file")
+        s = space.text_w(w_mode)
+        
+        for char in s:
+            if char in "+abx":
+                # OK do nothing
+                pass
+            else if char == "r":
+                if rwa:
+                    raise oefmt(space.w_ValueError,
+                            "invalid mode: %.200s", mode)
+                rwa = True
+                self.readable = True
+                if console_type == "x":
+                    console_type = "r"
+            else if char == "w":
+                if rwa:
+                    raise oefmt(space.w_ValueError,
+                            "invalid mode: %.200s", mode)
+                rwa = True
+                self.writable = True;
+                if console_type == 'x':
+                    console_type = 'w'
+            else:
+                raise oefmt(space.w_ValueError,
+                            "invalid mode: %.200s", mode)
+        if not rwa:
+            raise oefmt(space.w_ValueError,
+                        "Must have exactly one of read or write mode")
+        
+        if self.fd >= 0:
+            self.handle = rwin32.get_osfhandle(self.fd)
+            self.closehandle = False
+        else:
+            access = rwin32.GENERIC_READ
diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py
--- a/rpython/rlib/rwin32.py
+++ b/rpython/rlib/rwin32.py
@@ -137,6 +137,11 @@
     HMODULE = HANDLE
     NULL_HANDLE = rffi.cast(HANDLE, 0)
     INVALID_HANDLE_VALUE = rffi.cast(HANDLE, -1)
+    GENERIC_READ   =  rffi.cast(DWORD, 0x80000000)
+    GENERIC_WRITE  =  rffi.cast(DWORD, 0x40000000)
+    GENERIC_EXECUTE=  rffi.cast(DWORD, 0x20000000)
+    GENERIC_ALL    =  rffi.cast(DWORD, 0x10000000)
+
     PFILETIME = rffi.CArrayPtr(FILETIME)
 
     _GetLastError = winexternal('GetLastError', [], DWORD,

From pypy.commits at gmail.com  Mon May 27 18:43:36 2019
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 27 May 2019 15:43:36 -0700 (PDT)
Subject: [pypy-commit] pypy optimizeopt-cleanup: Don't fallback to jumping
 to the preamble in compile_retrace(), 
Message-ID: <5cec6818.1c69fb81.f5787.390d@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: optimizeopt-cleanup
Changeset: r96707:5e0d762fe4fd
Date: 2019-05-25 03:02 +0100
http://bitbucket.org/pypy/pypy/changeset/5e0d762fe4fd/

Log:	Don't fallback to jumping to the preamble in compile_retrace(),
	since optimize_peeled_loop() already takes care of it. And simplify
	some code.

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -126,20 +126,18 @@
     log_noopt = False
 
     def __init__(self, trace, celltoken, state,
-                 call_pure_results=None, enable_opts=None,
-                 inline_short_preamble=True):
+                 call_pure_results=None, enable_opts=None):
         self.trace = trace
         self.celltoken = celltoken
         self.enable_opts = enable_opts
         self.state = state
         self.call_pure_results = call_pure_results
-        self.inline_short_preamble = inline_short_preamble
 
     def optimize(self, metainterp_sd, jitdriver_sd, optimizations):
         from rpython.jit.metainterp.optimizeopt.unroll import UnrollOptimizer
         opt = UnrollOptimizer(metainterp_sd, jitdriver_sd, optimizations)
-        return opt.optimize_peeled_loop(self.trace, self.celltoken, self.state,
-            self.call_pure_results, self.inline_short_preamble)
+        return opt.optimize_peeled_loop(
+            self.trace, self.celltoken, self.state, self.call_pure_results)
 
 def show_procedures(metainterp_sd, procedure=None, error=None):
     from rpython.conftest import option
@@ -363,20 +361,9 @@
         loop_info, loop_ops = loop_data.optimize_trace(
             metainterp_sd, jitdriver_sd, metainterp.box_names_memo)
     except InvalidLoop:
-        # Fall back on jumping directly to preamble
+        metainterp_sd.jitlog.trace_aborted()
         history.cut(cut)
-        history.record(rop.JUMP, jumpargs[:], None, descr=loop_jitcell_token)
-        loop_data = UnrolledLoopData(trace, loop_jitcell_token, start_state,
-                                     call_pure_results=call_pure_results,
-                                     enable_opts=enable_opts,
-                                     inline_short_preamble=False)
-        try:
-            loop_info, loop_ops = loop_data.optimize_trace(
-                metainterp_sd, jitdriver_sd, metainterp.box_names_memo)
-        except InvalidLoop:
-            metainterp_sd.jitlog.trace_aborted()
-            history.cut(cut)
-            return None
+        return None
 
     label_op = loop_info.label_op
     label_token = label_op.getdescr()
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -115,8 +115,7 @@
         self._clean_optimization_info(self._newoperations)
         return exported_state, self._newoperations
 
-    def optimize_peeled_loop(self, trace, celltoken, state,
-                             call_pure_results, inline_short_preamble=True):
+    def optimize_peeled_loop(self, trace, celltoken, state, call_pure_results):
         trace = trace.get_iter()
         try:
             label_args = self.optunroll.import_state(trace.inputargs, state)
@@ -155,18 +154,12 @@
                                                     state.virtual_state)
         label_op.setdescr(target_token)
 
-        if not inline_short_preamble:
-            self.optunroll.jump_to_preamble(celltoken, end_jump, info)
-            return (UnrollInfo(target_token, label_op, extra_same_as,
-                               self.quasi_immutable_deps),
-                    self._newoperations)
-
         try:
             new_virtual_state = self.optunroll.jump_to_existing_trace(
-                    end_jump, label_op, state.runtime_boxes, force_boxes=False)
+                end_jump, label_op, state.runtime_boxes, force_boxes=False)
         except InvalidLoop:
             # inlining short preamble failed, jump to preamble
-            self.optunroll.jump_to_preamble(celltoken, end_jump, info)
+            self.jump_to_preamble(celltoken, end_jump)
             return (UnrollInfo(target_token, label_op, extra_same_as,
                                self.quasi_immutable_deps),
                     self._newoperations)
@@ -176,12 +169,12 @@
             # to the preamble.
             try:
                 new_virtual_state = self.optunroll.jump_to_existing_trace(
-                        end_jump, label_op, state.runtime_boxes, force_boxes=True)
+                    end_jump, label_op, state.runtime_boxes, force_boxes=True)
             except InvalidLoop:
                 pass
 
         if new_virtual_state is not None:
-            self.optunroll.jump_to_preamble(celltoken, end_jump, info)
+            self.jump_to_preamble(celltoken, end_jump)
             return (UnrollInfo(target_token, label_op, extra_same_as,
                                self.quasi_immutable_deps),
                     self._newoperations)
@@ -209,7 +202,8 @@
         cell_token = jump_op.getdescr()
         assert isinstance(cell_token, JitCellToken)
         if not inline_short_preamble or len(cell_token.target_tokens) == 1:
-            return self.optunroll.jump_to_preamble(cell_token, jump_op, info)
+            self.jump_to_preamble(cell_token, jump_op)
+            return info, self._newoperations[:]
         # force all the information that does not go to the short
         # preamble at all
         self.flush()
@@ -219,7 +213,8 @@
             vs = self.optunroll.jump_to_existing_trace(jump_op, None, runtime_boxes,
                                              force_boxes=False)
         except InvalidLoop:
-            return self.optunroll.jump_to_preamble(cell_token, jump_op, info)
+            self.jump_to_preamble(cell_token, jump_op)
+            return info, self._newoperations[:]
         if vs is None:
             return info, self._newoperations[:]
         warmrunnerdescr = self.metainterp_sd.warmrunnerdesc
@@ -237,7 +232,8 @@
             if vs is None:
                 return info, self._newoperations[:]
             debug_print("Retrace count reached, jumping to preamble")
-            return self.optunroll.jump_to_preamble(cell_token, jump_op, info)
+            self.jump_to_preamble(cell_token, jump_op)
+            return info, self._newoperations[:]
         exported_state = self.optunroll.export_state(info.jump_op.getarglist(),
                                            info.inputargs, runtime_boxes,
                                            box_names_memo)
@@ -245,6 +241,12 @@
         self._clean_optimization_info(self._newoperations)
         return exported_state, self._newoperations
 
+    def jump_to_preamble(self, cell_token, jump_op):
+        assert cell_token.target_tokens[0].virtual_state is None
+        jump_op = jump_op.copy_and_change(
+            rop.JUMP, descr=cell_token.target_tokens[0])
+        self.send_extra_operation(jump_op)
+
 
 class OptUnroll(Optimization):
     """Unroll the loop into two iterations. The first one will
@@ -304,13 +306,6 @@
         label_op.initarglist(label_op.getarglist() + sb.used_boxes)
         return target_token
 
-    def jump_to_preamble(self, cell_token, jump_op, info):
-        assert cell_token.target_tokens[0].virtual_state is None
-        jump_op = jump_op.copy_and_change(rop.JUMP,
-                                          descr=cell_token.target_tokens[0])
-        self.optimizer.send_extra_operation(jump_op)
-        return info, self.optimizer._newoperations[:]
-
 
     def jump_to_existing_trace(self, jump_op, label_op, runtime_boxes, force_boxes=False):
         jitcelltoken = jump_op.getdescr()
@@ -346,20 +341,14 @@
                 continue
 
             short_preamble = target_token.short_preamble
-            try:
-                extra = self.inline_short_preamble(args + virtuals, args,
-                                    short_preamble, self.optimizer.patchguardop,
-                                    target_token, label_op)
-            except KeyError:
-                # SHOULD NOT OCCUR BUT DOES: WHY??  issue #2185
-                self.optimizer.metainterp_sd.logger_ops.log_short_preamble([],
-                    short_preamble, {})
-                raise
-
+            extra = self.inline_short_preamble(args + virtuals, args,
+                                short_preamble, self.optimizer.patchguardop,
+                                target_token, label_op)
             self.optimizer.send_extra_operation(jump_op.copy_and_change(rop.JUMP,
                                       args=args + extra,
                                       descr=target_token))
             return None # explicit because the return can be non-None
+
         return virtual_state
 
     def _map_args(self, mapping, arglist):

From pypy.commits at gmail.com  Mon May 27 18:43:37 2019
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 27 May 2019 15:43:37 -0700 (PDT)
Subject: [pypy-commit] pypy optimizeopt-cleanup: Don't pass the optimizer
 around unnecessarily
Message-ID: <5cec6819.1c69fb81.a10fb.b308@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: optimizeopt-cleanup
Changeset: r96708:3df8ad2225a4
Date: 2019-05-27 20:01 +0100
http://bitbucket.org/pypy/pypy/changeset/3df8ad2225a4/

Log:	Don't pass the optimizer around unnecessarily

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -141,7 +141,7 @@
                 constptr = optforce.optimizer.constant_fold(op)
                 op.set_forwarded(constptr)
                 self._is_virtual = False
-                self._force_elements_immutable(self.descr, constptr, optforce)
+                self._force_elements_immutable(self.descr, constptr, optforce.optimizer)
                 return constptr
             #
             op.set_forwarded(None)
@@ -302,12 +302,12 @@
                     return False    # not a constant at all
         return True
 
-    def _force_elements_immutable(self, descr, constptr, optforce):
+    def _force_elements_immutable(self, descr, constptr, optimizer):
         for i, fielddescr in enumerate(descr.get_all_fielddescrs()):
             fld = self._fields[i]
-            subbox = optforce.optimizer.force_box(fld)
+            subbox = optimizer.force_box(fld)
             assert isinstance(subbox, Const)
-            execute(optforce.optimizer.cpu, None, rop.SETFIELD_GC,
+            execute(optimizer.cpu, None, rop.SETFIELD_GC,
                     fielddescr, constptr, subbox)
 
 class InstancePtrInfo(AbstractStructPtrInfo):
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -703,12 +703,12 @@
     def already_seen_virtual(self, keybox):
         return keybox in self.fieldboxes
 
-    def create_state_or_none(self, box, opt):
+    def create_state_or_none(self, box):
         if box is None:
             return None
-        return self.create_state(box, opt)
+        return self.create_state(box)
 
-    def create_state(self, box, opt):
+    def create_state(self, box):
         box = get_box_replacement(box)
         try:
             return self.info[box]
@@ -720,7 +720,7 @@
                 result = info.visitor_dispatch_virtual_type(self)
                 self.info[box] = result
                 info.visitor_walk_recursive(box, self)
-                result.fieldstate = [self.create_state_or_none(b, opt)
+                result.fieldstate = [self.create_state_or_none(b)
                                      for b in self.fieldboxes[box]]
             else:
                 result = self.visit_not_virtual(box)
@@ -733,14 +733,10 @@
         return result
 
     def get_virtual_state(self, jump_args):
-        if self.optimizer.optearlyforce:
-            opt = self.optimizer.optearlyforce
-        else:
-            opt = self.optimizer
         state = []
         self.info = {}
         for box in jump_args:
-            state.append(self.create_state(box, opt))
+            state.append(self.create_state(box))
         return VirtualState(state)
 
     def visit_not_virtual(self, box):
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -256,7 +256,7 @@
         numb_state.num_boxes = num_boxes
         numb_state.num_virtuals = num_virtuals
 
-    def number(self, optimizer, position, trace):
+    def number(self, position, trace):
         snapshot_iter = trace.get_snapshot_iter(position)
         numb_state = NumberingState(snapshot_iter.size)
         numb_state.append_int(0) # patch later: size of resume section
@@ -390,7 +390,7 @@
         fieldboxes = []
         for box in _fieldboxes:
             if box is not None:
-                box = self.optimizer.get_box_replacement(box)
+                box = box.get_box_replacement()
             fieldboxes.append(box)
         self.vfieldboxes[virtualbox] = fieldboxes
         self._register_boxes(fieldboxes)
@@ -417,7 +417,6 @@
     def finish(self, pending_setfields=[]):
         from rpython.jit.metainterp.optimizeopt.info import (
             getrawptrinfo, getptrinfo)
-        optimizer = self.optimizer
         # compute the numbering
         storage = self.storage
         # make sure that nobody attached resume data to this guard yet
@@ -425,8 +424,7 @@
         resume_position = self.guard_op.rd_resume_position
         assert resume_position >= 0
         # count stack depth
-        numb_state = self.memo.number(optimizer,
-            resume_position, optimizer.trace)
+        numb_state = self.memo.number(resume_position, self.trace)
         self.liveboxes_from_env = liveboxes_from_env = numb_state.liveboxes
         num_virtuals = numb_state.num_virtuals
         self.liveboxes = {}
@@ -451,12 +449,14 @@
 
         for setfield_op in pending_setfields:
             box = setfield_op.getarg(0)
-            box = optimizer.get_box_replacement(box)
+            if box is not None:
+                box = box.get_box_replacement()
             if setfield_op.getopnum() == rop.SETFIELD_GC:
                 fieldbox = setfield_op.getarg(1)
             else:
                 fieldbox = setfield_op.getarg(2)
-            fieldbox = optimizer.get_box_replacement(fieldbox)
+            if fieldbox is not None:
+                fieldbox = fieldbox.get_box_replacement()
             self.register_box(box)
             self.register_box(fieldbox)
             info = getptrinfo(fieldbox)
@@ -464,7 +464,7 @@
             info.visitor_walk_recursive(fieldbox, self)
 
         self._number_virtuals(liveboxes, num_virtuals)
-        self._add_pending_fields(optimizer, pending_setfields)
+        self._add_pending_fields(pending_setfields)
 
         numb_state.patch(1, len(liveboxes))
 
@@ -519,8 +519,7 @@
                 info = getptrinfo(virtualbox)
                 assert info.is_virtual()
                 assert isinstance(info, AbstractVirtualPtrInfo)
-                fieldnums = [self._gettagged(box)
-                             for box in fieldboxes]
+                fieldnums = [self._gettagged(box) for box in fieldboxes]
                 vinfo = self.make_virtual_info(info, fieldnums)
                 # if a new vinfo instance is made, we get the fieldnums list we
                 # pass in as an attribute. hackish.
@@ -540,19 +539,21 @@
                 return True
         return False
 
-    def _add_pending_fields(self, optimizer, pending_setfields):
+    def _add_pending_fields(self, pending_setfields):
+        from rpython.jit.metainterp.optimizeopt.util import (
+            get_box_replacement)
         rd_pendingfields = lltype.nullptr(PENDINGFIELDSP.TO)
         if pending_setfields:
             n = len(pending_setfields)
             rd_pendingfields = lltype.malloc(PENDINGFIELDSP.TO, n)
             for i in range(n):
                 op = pending_setfields[i]
-                box = optimizer.get_box_replacement(op.getarg(0))
+                box = get_box_replacement(op.getarg(0))
                 descr = op.getdescr()
                 opnum = op.getopnum()
                 if opnum == rop.SETARRAYITEM_GC:
                     fieldbox = op.getarg(2)
-                    boxindex = optimizer.get_box_replacement(op.getarg(1))
+                    boxindex = op.getarg(1).get_box_replacement()
                     itemindex = boxindex.getint()
                     # sanity: it's impossible to run code with SETARRAYITEM_GC
                     # with negative index, so this guard cannot ever fail;
@@ -564,8 +565,7 @@
                     itemindex = -1
                 else:
                     raise AssertionError
-                fieldbox = optimizer.get_box_replacement(fieldbox)
-                #descr, box, fieldbox, itemindex = pending_setfields[i]
+                fieldbox = get_box_replacement(fieldbox)
                 lldescr = annlowlevel.cast_instance_to_base_ptr(descr)
                 num = self._gettagged(box)
                 fieldnum = self._gettagged(fieldbox)
diff --git a/rpython/jit/metainterp/test/test_resume.py b/rpython/jit/metainterp/test/test_resume.py
--- a/rpython/jit/metainterp/test/test_resume.py
+++ b/rpython/jit/metainterp/test/test_resume.py
@@ -836,7 +836,7 @@
 
     iter = t.get_iter()
     b1, b2, b3, b4, b5 = iter.inputargs
-    numb_state = memo.number(FakeOptimizer(), 0, iter)
+    numb_state = memo.number(0, iter)
     numb = numb_state.create_numbering()
     assert numb_state.num_virtuals == 0
 
@@ -852,7 +852,7 @@
                                   False, [], [])
     snap2.prev = snap
 
-    numb_state2 = memo.number(FakeOptimizer(), 1, iter)
+    numb_state2 = memo.number(1, iter)
     numb2 = numb_state2.create_numbering()
     assert numb_state2.num_virtuals == 0
 
@@ -876,7 +876,7 @@
 
     # renamed
     b3.set_forwarded(c4)
-    numb_state3 = memo.number(FakeOptimizer(), 2, iter)
+    numb_state3 = memo.number(2, iter)
     numb3 = numb_state3.create_numbering()
     assert numb_state3.num_virtuals == 0
 
@@ -892,7 +892,7 @@
     snap4.prev = snap
 
     b4.set_forwarded(FakeVirtualInfo(True))
-    numb_state4 = memo.number(FakeOptimizer(), 3, iter)
+    numb_state4 = memo.number(3, iter)
     numb4 = numb_state4.create_numbering()
     assert numb_state4.num_virtuals == 1
 
@@ -911,7 +911,7 @@
 
     b4.set_forwarded(FakeVirtualInfo(True))
     b5.set_forwarded(FakeVirtualInfo(True))
-    numb_state5 = memo.number(FakeOptimizer(), 4, iter)
+    numb_state5 = memo.number(4, iter)
     numb5 = numb_state5.create_numbering()
     assert numb_state5.num_virtuals == 2
 
@@ -923,8 +923,9 @@
         2, 1, tag(3, TAGINT), tag(0, TAGVIRTUAL), tag(0, TAGBOX), tag(3, TAGINT)
         ] + [0, 0]
 
- at given(strategies.lists(strategies.builds(IntFrontendOp, strategies.just(0)) | intconsts,
-       min_size=1))
+ at given(strategies.lists(
+    strategies.builds(IntFrontendOp, strategies.just(0)) | intconsts,
+    min_size=1))
 def test_ResumeDataLoopMemo_random(lst):
     inpargs = [box for box in lst if not isinstance(box, Const)]
     metainterp_sd = FakeMetaInterpStaticData()
@@ -933,7 +934,7 @@
     i = t.get_iter()
     t.create_top_snapshot(FakeJitCode("", 0), 0, Frame(lst), False, [], [])
     memo = ResumeDataLoopMemo(metainterp_sd)
-    numb_state = memo.number(FakeOptimizer(), 0, i)
+    numb_state = memo.number(0, i)
     numb = numb_state.create_numbering()
     l = unpack_numbering(numb)
     assert l[0] == len(l)
@@ -1386,7 +1387,7 @@
     liveboxes = []
     modifier._number_virtuals(liveboxes, 0)
     assert liveboxes == [b2s, b4s] or liveboxes == [b4s, b2s]
-    modifier._add_pending_fields(FakeOptimizer(), [
+    modifier._add_pending_fields([
         ResOperation(rop.SETFIELD_GC, [b2s, b4s], descr=LLtypeMixin.nextdescr)])
     storage.rd_consts = memo.consts[:]
     storage.rd_numb = Numbering([0])
@@ -1415,7 +1416,7 @@
         pass
     storage = Storage()
     modifier = ResumeDataVirtualAdder(None, storage, storage, None, None)
-    modifier._add_pending_fields(None, [])
+    modifier._add_pending_fields([])
     assert not storage.rd_pendingfields
     #
     class FieldDescr(AbstractDescr):
@@ -1428,9 +1429,8 @@
     b = IntFrontendOp(0)
     modifier.liveboxes_from_env = {a: rffi.cast(rffi.SHORT, 1042),
                                    b: rffi.cast(rffi.SHORT, 1061)}
-    modifier._add_pending_fields(FakeOptimizer(), [
-        ResOperation(rop.SETFIELD_GC, [a, b],
-                     descr=field_a)])
+    modifier._add_pending_fields(
+        [ResOperation(rop.SETFIELD_GC, [a, b], descr=field_a)])
     pf = storage.rd_pendingfields
     assert len(pf) == 1
     assert (annlowlevel.cast_base_ptr_to_instance(FieldDescr, pf[0].lldescr)
@@ -1450,7 +1450,7 @@
                                    a61: rffi.cast(rffi.SHORT, 1061),
                                    a62: rffi.cast(rffi.SHORT, 1062),
                                    a63: rffi.cast(rffi.SHORT, 1063)}
-    modifier._add_pending_fields(FakeOptimizer(), [
+    modifier._add_pending_fields([
         ResOperation(rop.SETARRAYITEM_GC, [a42, ConstInt(0), a61],
                      descr=array_a),
         ResOperation(rop.SETARRAYITEM_GC, [a42, ConstInt(2147483647), a62],
@@ -1469,11 +1469,10 @@
     assert rffi.cast(lltype.Signed, pf[1].itemindex) == 2147483647
     #
     if sys.maxint >= 2147483648:
-        py.test.raises(TagOverflow, modifier._add_pending_fields,
-                       FakeOptimizer(),
-                       [ResOperation(rop.SETARRAYITEM_GC,
-                                     [a42, ConstInt(2147483648), a63],
-                                     descr=array_a)])
+        with py.test.raises(TagOverflow):
+            modifier._add_pending_fields(
+                [ResOperation(rop.SETARRAYITEM_GC,
+                    [a42, ConstInt(2147483648), a63], descr=array_a)])
 
 def test_resume_reader_fields_and_arrayitems():
     class ResumeReader(AbstractResumeDataReader):

From pypy.commits at gmail.com  Tue May 28 02:18:32 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 27 May 2019 23:18:32 -0700 (PDT)
Subject: [pypy-commit] pypy default: require binary vmprof since building
 from source requires libunwind on linux
Message-ID: <5cecd2b8.1c69fb81.86df5.93d7@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r96709:bc4cf55dc154
Date: 2019-05-28 09:14 +0300
http://bitbucket.org/pypy/pypy/changeset/bc4cf55dc154/

Log:	require binary vmprof since building from source requires libunwind
	on linux

diff --git a/requirements.txt b/requirements.txt
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,5 @@
+--only-binary vmprof
+
 cffi>=1.4.0
 
 # parse log files in rvmprof tests

From pypy.commits at gmail.com  Tue May 28 02:18:34 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 27 May 2019 23:18:34 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: merge default into branch
Message-ID: <5cecd2ba.1c69fb81.9e194.f524@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r96710:018e437e5061
Date: 2019-05-28 09:16 +0300
http://bitbucket.org/pypy/pypy/changeset/018e437e5061/

Log:	merge default into branch

diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -425,9 +425,11 @@
         setarrayitem_raw(i153, 0, i106, descr=...)
         p156 = getfield_gc_r(p48, descr=...)
         i158 = getfield_raw_i(..., descr=...)
+        i160 = int_sub(i158, 16)
+        setfield_raw(#, i160, descr=...)
         setfield_gc(p48, p49, descr=...)
         setfield_gc(p134, ConstPtr(null), descr=...)
-        i160 = int_lt(i158, 0)
+        i160 = int_lt(i160, 0)
         guard_false(i160, descr=...)
         jump(..., descr=...)
         """)
diff --git a/requirements.txt b/requirements.txt
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,5 @@
+--only-binary vmprof
+
 cffi>=1.4.0
 
 # parse log files in rvmprof tests
diff --git a/rpython/tool/jitlogparser/parser.py b/rpython/tool/jitlogparser/parser.py
--- a/rpython/tool/jitlogparser/parser.py
+++ b/rpython/tool/jitlogparser/parser.py
@@ -504,8 +504,10 @@
     for line in lines:
         if line:
             num, count = line.split(':', 2)
-            mapping[num].count = int(count)
-
+            try:
+                mapping[num].count = int(count)
+            except KeyError:
+                pass # too bad
 
 def mangle_descr(descr):
     if descr.startswith('TargetToken('):
diff --git a/testrunner/get_info.py b/testrunner/get_info.py
--- a/testrunner/get_info.py
+++ b/testrunner/get_info.py
@@ -10,8 +10,7 @@
 BASE_DIR = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 if sys.platform.startswith('win'):
     TARGET_NAME = r'pypy3-c.exe'
-    # see https://github.com/pypa/virtualenv/issues/789
-    TARGET_DIR = 'bin'
+    TARGET_DIR = 'Scripts'
 else:
     TARGET_NAME = 'pypy3-c'
     TARGET_DIR = 'bin'

From pypy.commits at gmail.com  Tue May 28 06:41:07 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 28 May 2019 03:41:07 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix bug: when a newdict(instance=True)
 was switching its strategy, it lost its
Message-ID: <5ced1043.1c69fb81.d2103.46ab@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r96711:e19e79b1385f
Date: 2019-05-28 12:22 +0200
http://bitbucket.org/pypy/pypy/changeset/e19e79b1385f/

Log:	fix bug: when a newdict(instance=True) was switching its strategy,
	it lost its content

diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -78,8 +78,9 @@
             W_ModuleDictObject.__init__(w_obj, space, strategy, storage)
             return w_obj
         elif instance:
-            from pypy.objspace.std.mapdict import MapDictStrategy
-            strategy = space.fromcache(MapDictStrategy)
+            from pypy.objspace.std.mapdict import make_instance_dict
+            assert w_type is None
+            return make_instance_dict(space)
         elif strdict or module:
             assert w_type is None
             strategy = space.fromcache(BytesDictStrategy)
diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -753,6 +753,7 @@
         self.space = space
 
     def get_empty_storage(self):
+        # mainly used for tests
         w_result = Object()
         terminator = self.space.fromcache(get_terminator_for_dicts)
         w_result._mapdict_init_empty(terminator)
@@ -865,6 +866,11 @@
     def iteritems(self, w_dict):
         return MapDictIteratorItems(self.space, self, w_dict)
 
+def make_instance_dict(space):
+    w_fake_object = Object()
+    terminator = space.fromcache(get_terminator_for_dicts)
+    w_fake_object._mapdict_init_empty(terminator)
+    return w_fake_object.getdict(space)
 
 def materialize_r_dict(space, obj, dict_w):
     map = obj._get_mapdict_map()
diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py
--- a/pypy/objspace/std/test/test_mapdict.py
+++ b/pypy/objspace/std/test/test_mapdict.py
@@ -897,6 +897,17 @@
         d = x.__dict__
         assert list(__pypy__.reversed_dict(d)) == d.keys()[::-1]
 
+    def test_bug_materialize_huge_dict(self):
+        import __pypy__
+        d = __pypy__.newdict("instance")
+        for i in range(100):
+            d[str(i)] = i
+        assert len(d) == 100
+
+        for key in d:
+            assert d[key] == int(key)
+
+
 
 class AppTestWithMapDictAndCounters(object):
     spaceconfig = {"objspace.std.withmethodcachecounter": True}

From pypy.commits at gmail.com  Wed May 29 11:17:48 2019
From: pypy.commits at gmail.com (fijal)
Date: Wed, 29 May 2019 08:17:48 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: my talk from today
Message-ID: <5ceea29c.1c69fb81.babf9.fd34@mx.google.com>

Author: fijal
Branch: extradoc
Changeset: r5947:e7f187d7eefa
Date: 2019-05-29 17:17 +0200
http://bitbucket.org/pypy/extradoc/changeset/e7f187d7eefa/

Log:	my talk from today

diff --git a/talk/synerise2019/talk.key b/talk/synerise2019/talk.key
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..5bb346ed50268b9dd456d2cc1505d2f05caab4d1
GIT binary patch

[cut]

diff --git a/talk/synerise2019/talk.pdf b/talk/synerise2019/talk.pdf
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..3745b05f0578ffeb7c1a931dd1c4110bede1e840
GIT binary patch

[cut]


From pypy.commits at gmail.com  Wed May 29 14:40:07 2019
From: pypy.commits at gmail.com (mattip)
Date: Wed, 29 May 2019 11:40:07 -0700 (PDT)
Subject: [pypy-commit] pypy cffi-libs2: add cffi implementation of crypt
Message-ID: <5ceed207.1c69fb81.72346.095f@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cffi-libs2
Changeset: r96712:4e4d2be6ff04
Date: 2019-05-29 20:55 +0300
http://bitbucket.org/pypy/pypy/changeset/4e4d2be6ff04/

Log:	add cffi implementation of crypt

diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -176,7 +176,7 @@
     RegrTest('test_copy_reg.py', core=True),
     RegrTest('test_cpickle.py', core=True),
     RegrTest('test_cprofile.py'),
-    RegrTest('test_crypt.py', usemodules='crypt'),
+    RegrTest('test_crypt.py'),
     RegrTest('test_csv.py', usemodules='_csv'),
     RegrTest('test_ctypes.py', usemodules="_rawffi thread cpyext"),
     RegrTest('test_curses.py'),
diff --git a/lib_pypy/crypt/__init__.py b/lib_pypy/crypt/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/crypt/__init__.py
@@ -0,0 +1,21 @@
+"""
+CFFI based implementation of the crypt module
+"""
+
+import sys
+import cffi
+
+ffi = cffi.FFI()
+ffi.cdef('char *crypt(char *word, char *salt);')
+
+try:
+    lib = ffi.dlopen('crypt')
+except OSError:
+    raise ImportError('crypt not available')
+
+
+def crypt(word, salt):
+    res = lib.crypt(word, salt)
+    if not res:
+        return None
+    return ffi.string(res)
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -31,13 +31,13 @@
 working_modules = default_modules.copy()
 working_modules.update([
     "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd",
-    "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios",
+    "select", "zipimport", "_lsprof", "signal", "_rawffi", "termios",
     "zlib", "bz2", "struct", "_md5", "_sha", "_minimal_curses",
     "cStringIO", "thread", "itertools", "pyexpat", "cpyext", "array",
     "binascii", "_multiprocessing", '_warnings', "_collections",
     "_multibytecodec", "micronumpy", "_continuation", "_cffi_backend",
     "_csv", "_cppyy", "_pypyjson", "_jitlog",
-    #" _ssl", "_hashlib"
+    #" _ssl", "_hashlib", "crypt"
 ])
 
 import rpython.rlib.rvmprof.cintf

From pypy.commits at gmail.com  Thu May 30 00:44:53 2019
From: pypy.commits at gmail.com (mattip)
Date: Wed, 29 May 2019 21:44:53 -0700 (PDT)
Subject: [pypy-commit] pypy default: be more tolerant of possible missing
 names from working_modules
Message-ID: <5cef5fc5.1c69fb81.cdc73.5a1f@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r96716:0d84a7a0a0fe
Date: 2019-05-30 07:42 +0300
http://bitbucket.org/pypy/pypy/changeset/0d84a7a0a0fe/

Log:	be more tolerant of possible missing names from working_modules

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -65,7 +65,8 @@
     working_modules.add("_winreg")
     # unix only modules
     for name in ["crypt", "fcntl", "pwd", "termios", "_minimal_curses"]:
-        working_modules.remove(name)
+        if name in working_modules:
+            working_modules.remove(name)
         if name in translation_modules:
             translation_modules.remove(name)
 

From pypy.commits at gmail.com  Thu May 30 00:44:55 2019
From: pypy.commits at gmail.com (mattip)
Date: Wed, 29 May 2019 21:44:55 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: merge default into branch
Message-ID: <5cef5fc7.1c69fb81.3b58b.708c@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r96717:3183fe48a614
Date: 2019-05-30 07:44 +0300
http://bitbucket.org/pypy/pypy/changeset/3183fe48a614/

Log:	merge default into branch

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -69,7 +69,8 @@
     # unix only modules
     for name in ["crypt", "fcntl", "pwd", "termios", "_minimal_curses",
                  "_posixsubprocess"]:
-        working_modules.remove(name)
+        if name in working_modules:
+            working_modules.remove(name)
         if name in translation_modules:
             translation_modules.remove(name)
 

From pypy.commits at gmail.com  Thu May 30 16:15:24 2019
From: pypy.commits at gmail.com (mattip)
Date: Thu, 30 May 2019 13:15:24 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: skip test that only works translated
Message-ID: <5cf039dc.1c69fb81.87093.f996@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r96718:51716b155429
Date: 2019-05-30 15:49 +0300
http://bitbucket.org/pypy/pypy/changeset/51716b155429/

Log:	skip test that only works translated

diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -436,6 +436,9 @@
 class AppTestPartialEvaluation:
     spaceconfig = dict(usemodules=['array',])
 
+    def setup_class(cls):
+        cls.w_appdirect = cls.space.wrap(cls.runappdirect)
+
     def test_partial_utf8(self):
         import _codecs
         encoding = 'utf-8'
@@ -1404,6 +1407,8 @@
     def test_invalid_type_errors(self):
         # hex is not a text encoding. it works via the codecs functions, but
         # not the methods
+        if not self.appdirect:
+            skip('"hex" only available after translation')
         import codecs
         res = codecs.decode(b"aabb", "hex")
         assert res == b"\xaa\xbb"

From pypy.commits at gmail.com  Thu May 30 16:15:26 2019
From: pypy.commits at gmail.com (mattip)
Date: Thu, 30 May 2019 13:15:26 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: win32, python2 doesn't like surrogates,
 skip this check
Message-ID: <5cf039de.1c69fb81.ccaf1.bb1d@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r96719:8c02125cb1c5
Date: 2019-05-30 16:51 +0300
http://bitbucket.org/pypy/pypy/changeset/8c02125cb1c5/

Log:	win32, python2 doesn't like surrogates, skip this check

diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -1,5 +1,7 @@
 """The builtin str implementation"""
 
+import sys
+
 from rpython.rlib.objectmodel import (
     compute_hash, compute_unique_id, import_from_mixin, always_inline,
     enforceargs, newlist_hint, specialize, we_are_translated)
@@ -38,7 +40,7 @@
         self._utf8 = utf8str
         self._length = length
         self._index_storage = rutf8.null_storage()
-        if not we_are_translated():
+        if not we_are_translated() and not sys.platform == 'win32':
             # utf8str must always be a valid utf8 string, except maybe with
             # explicit surrogate characters---which .decode('utf-8') doesn't
             # special-case in Python 2, which is exactly what we want here

From pypy.commits at gmail.com  Thu May 30 17:10:15 2019
From: pypy.commits at gmail.com (mattip)
Date: Thu, 30 May 2019 14:10:15 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: fixes for win32
Message-ID: <5cf046b7.1c69fb81.69dbe.5976@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r96720:95ef57b94c9d
Date: 2019-05-31 00:09 +0300
http://bitbucket.org/pypy/pypy/changeset/95ef57b94c9d/

Log:	fixes for win32

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -2277,7 +2277,7 @@
                                space.newint(info[2])])
 
     def _getfinalpathname(space, w_path):
-        path = space.utf8_w(w_path)
+        path = space.utf8_w(w_path).decode('utf-8')
         try:
             result = nt._getfinalpathname(path)
         except nt.LLNotImplemented as e:
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -10,7 +10,6 @@
 from pypy.interpreter.gateway import interp2app
 from rpython.translator.c.test.test_extfunc import need_sparse_files
 from rpython.rlib import rposix
-from rpython.rlib.objectmodel import we_are_translated
 
 USEMODULES = ['binascii', 'posix', 'signal', 'struct', 'time']
 # py3k os.open uses subprocess, requiring the following per platform
@@ -319,7 +318,7 @@
         ex(self.posix.write, UNUSEDFD, b"x")
         ex(self.posix.close, UNUSEDFD)
         #UMPF cpython raises IOError ex(self.posix.ftruncate, UNUSEDFD, 123)
-        if sys.platform == 'win32' and not we_are_translated():
+        if sys.platform == 'win32' and self.runappdirect:
             # XXX kills the host interpreter untranslated
             ex(self.posix.fstat, UNUSEDFD)
             ex(self.posix.stat, "qweqwehello")
@@ -1082,7 +1081,7 @@
         os.closerange(start, stop)
         for fd in fds:
             os.close(fd)     # should not have been closed
-        if self.platform == 'win32' and not we_are_translated():
+        if self.platform == 'win32' and self.runappdirect:
             # XXX kills the host interpreter untranslated
             for fd in range(start, stop):
                 raises(OSError, os.fstat, fd)   # should have been closed
diff --git a/pypy/module/posix/test/test_scandir.py b/pypy/module/posix/test/test_scandir.py
--- a/pypy/module/posix/test/test_scandir.py
+++ b/pypy/module/posix/test/test_scandir.py
@@ -189,7 +189,7 @@
     def test_fspath(self):
         posix = self.posix
         d = next(posix.scandir(self.dir1))
-        assert self.posix.fspath(d).endswith('dir1/file1')
+        assert self.posix.fspath(d).endswith('dir1' + self.sep + 'file1')
 
     def test_resource_warning(self):
         posix = self.posix

From pypy.commits at gmail.com  Thu May 30 17:13:35 2019
From: pypy.commits at gmail.com (mattip)
Date: Thu, 30 May 2019 14:13:35 -0700 (PDT)
Subject: [pypy-commit] pypy win32-encoding: start to use
 filesystemencoderrors
Message-ID: <5cf0477f.1c69fb81.fe733.d2d4@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-encoding
Changeset: r96721:51a97447c69a
Date: 2019-05-30 11:20 +0300
http://bitbucket.org/pypy/pypy/changeset/51a97447c69a/

Log:	start to use filesystemencoderrors

diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -25,6 +25,7 @@
         self.recursionlimit = 1000
         self.defaultencoding = "utf-8"
         self.filesystemencoding = None
+        self.filesystemencoderrors = None
         self.debug = True
         self.track_resources = False
         self.finalizing = False
@@ -152,16 +153,27 @@
         space = self.space
 
         if not space.config.translating:
-            ##from pypy.module.sys.interp_encoding import _getfilesystemencoding
-            ##self.filesystemencoding = _getfilesystemencoding(space)
-            # XXX the two lines above take a few seconds to run whenever
-            # we initialize the space; for tests, use a simpler version.
-            # Check what exactly breaks, if anything, in py3.5.  This is
-            # not strictly necessary but is an extremely nice-to-have
-            # feature: running just one test for example take 3.5
-            # seconds instead of 11.
-            from pypy.module.sys.interp_encoding import base_encoding
-            self.filesystemencoding = base_encoding
+            if sys.platform == 'win32':
+                legacywindowsfsencodingflag = 0
+                # TODO: read os.environ.get('PYTHONLEGACYWINDOWSFSENCODING', 0)
+                if legacywindowsfsencodingflag:
+                    self.filesystemencoding = 'mbcs'
+                    self.filesystemencoderrors = 'replace'
+                else:
+                    self.filesystemencoding = 'utf-8'
+                    self.filesystemencoderrors = 'surrogatepass'
+            else:
+                ##from pypy.module.sys.interp_encoding import _getfilesystemencoding
+                ##self.filesystemencoding = _getfilesystemencoding(space)
+                # XXX the two lines above take a few seconds to run whenever
+                # we initialize the space; for tests, use a simpler version.
+                # Check what exactly breaks, if anything, in py3.  This is
+                # not strictly necessary but is an extremely nice-to-have
+                # feature: running just one test for example take 3.5
+                # seconds instead of 11.
+                from pypy.module.sys.interp_encoding import base_encoding
+                self.filesystemencoding = base_encoding
+                self.filesystemencoderrors = 'surrogateescape'
 
             # Set up sys.prefix and friends, like app_main.py would do
             # We somewhat arbitrarily use the repo's root dir as sys.prefix
diff --git a/pypy/module/sys/interp_encoding.py b/pypy/module/sys/interp_encoding.py
--- a/pypy/module/sys/interp_encoding.py
+++ b/pypy/module/sys/interp_encoding.py
@@ -1,6 +1,5 @@
 import sys
 from rpython.rlib import rlocale
-from rpython.rlib.objectmodel import we_are_translated
 
 def getdefaultencoding(space):
     """Return the current default string encoding used by the Unicode
@@ -8,7 +7,8 @@
     return space.newtext(space.sys.defaultencoding)
 
 if sys.platform == "win32":
-    base_encoding = "mbcs"
+    # crash, this should not be used
+    base_encoding = "utf-8"
 elif sys.platform == "darwin":
     base_encoding = "utf-8"
 elif sys.platform == "linux2":
@@ -51,4 +51,6 @@
 
 
 def getfilesystemencodeerrors(space):
-    return space.newtext('surrogateescape')
+    if space.sys.filesystemencoderrors is None:
+        return space.newtext('surrogateescape')
+    return space.newtext(space.sys.filesystemencoderrors)

From pypy.commits at gmail.com  Fri May 31 04:41:16 2019
From: pypy.commits at gmail.com (mattip)
Date: Fri, 31 May 2019 01:41:16 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: properly use fsencode on obscure
 nt._getfinalpathname, fixes translation
Message-ID: <5cf0e8ac.1c69fb81.86927.952b@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r96722:1e8797906e74
Date: 2019-05-31 11:40 +0300
http://bitbucket.org/pypy/pypy/changeset/1e8797906e74/

Log:	properly use fsencode on obscure nt._getfinalpathname, fixes
	translation

diff --git a/pypy/module/posix/interp_nt.py b/pypy/module/posix/interp_nt.py
--- a/pypy/module/posix/interp_nt.py
+++ b/pypy/module/posix/interp_nt.py
@@ -1,5 +1,6 @@
 from rpython.rlib import rwin32
 from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rwin32file import make_win32_traits
 from rpython.rlib._os_support import UnicodeTraits
 from rpython.translator import cdir
@@ -91,6 +92,7 @@
     assert traits.str is unicode, 'Currently only handles unicode paths'
     win32traits = make_traits(traits)
 
+    @specialize.argtype(0)
     def _getfinalpathname_llimpl(path):
         if not win32traits.check_GetFinalPathNameByHandle():
             raise LLNotImplemented("GetFinalPathNameByHandle not available on "
@@ -122,7 +124,8 @@
                     VOLUME_NAME_DOS)
                 if result == 0:
                     raise rwin32.lastSavedWindowsError("GetFinalPathNameByHandle")
-                return buf.str(rffi.cast(lltype.Signed, result))
+                res = buf.str(rffi.cast(lltype.Signed, result))
+                return res.encode('utf8'), len(res)
         finally:
             rwin32.CloseHandle(hFile)
 
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -2277,15 +2277,14 @@
                                space.newint(info[2])])
 
     def _getfinalpathname(space, w_path):
-        path = space.utf8_w(w_path).decode('utf-8')
         try:
-            result = nt._getfinalpathname(path)
+            s, lgt = dispatch_filename(nt._getfinalpathname)(space, w_path)
         except nt.LLNotImplemented as e:
             raise OperationError(space.w_NotImplementedError,
                                  space.newtext(e.msg))
         except OSError as e:
             raise wrap_oserror2(space, e, w_path, eintr_retry=False)
-        return space.newtext(result)
+        return space.newtext(s, lgt)
 
 
 def chflags():
diff --git a/pypy/module/posix/test/test_nt.py b/pypy/module/posix/test/test_nt.py
--- a/pypy/module/posix/test/test_nt.py
+++ b/pypy/module/posix/test/test_nt.py
@@ -19,9 +19,9 @@
 
 
 def test__getfinalpathname():
-    path = __file__.decode('mbcs')
+    path = __file__.decode('utf-8')
     try:
-        result = nt._getfinalpathname(path)
+        result, lgt = nt._getfinalpathname(path)
     except nt.LLNotImplemented:
         py.test.skip("_getfinalpathname not supported on this platform")
     assert os.path.exists(result)
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -383,7 +383,7 @@
     def newbytearray(self, l):
         return W_BytearrayObject(l)
 
-    # XXX TODO - remove this and force all users to call with utf8
+    # XXX TODO - remove the specialization and force all users to call with utf8
     @specialize.argtype(1)
     def newtext(self, s, lgt=-1, unused=-1):
         # the unused argument can be from something like

From pypy.commits at gmail.com  Fri May 31 09:05:30 2019
From: pypy.commits at gmail.com (stevie_92)
Date: Fri, 31 May 2019 06:05:30 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-gc-cycle: Fixed some issues with
 tracking/deallocating cpyext gc objects
Message-ID: <5cf1269a.1c69fb81.cae8c.a5d2@mx.google.com>

Author: Stefan Beyer <home at sbeyer.at>
Branch: cpyext-gc-cycle
Changeset: r96723:0b73323d80cc
Date: 2019-05-30 15:11 +0200
http://bitbucket.org/pypy/pypy/changeset/0b73323d80cc/

Log:	Fixed some issues with tracking/deallocating cpyext gc objects Added
	rudimentary support for lists (gc issues, clear missing)

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1169,6 +1169,11 @@
         [rffi.VOIDP], lltype.Void,
         compilation_info=eci,
         _nowrapper=True)
+    state.C.PyObject_GC_Del = rffi.llexternal(
+        mangle_name(prefix, 'PyObject_GC_Del'),
+        [rffi.VOIDP], lltype.Void,
+        compilation_info=eci,
+        _nowrapper=True)
     state.C.PyType_GenericAlloc = rffi.llexternal(
         mangle_name(prefix, 'PyType_GenericAlloc'),
         [PyTypeObjectPtr, Py_ssize_t], PyObject,
@@ -1186,6 +1191,9 @@
         '_PyPy_tuple_free', [rffi.VOIDP], lltype.Void,
         compilation_info=eci, _nowrapper=True)
     from pypy.module.cpyext.typeobjectdefs import visitproc
+    state.C.PyObject_GC_Track = rffi.llexternal(
+        'PyObject_GC_Track', [rffi.VOIDP], lltype.Void,
+        compilation_info=eci, _nowrapper=True)
     state.C._PyPy_tuple_traverse = rffi.llexternal(
         '_PyPy_tuple_traverse', [PyObject, visitproc, rffi.VOIDP],
         rffi.INT, compilation_info=eci, _nowrapper=True)
diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -8,7 +8,6 @@
 from pypy.objspace.std.listobject import W_ListObject
 from pypy.interpreter.error import oefmt
 
-
 PyList_Check, PyList_CheckExact = build_type_checkers_flags("List")
 
 @cpython_api([Py_ssize_t], PyObject)
@@ -30,6 +29,33 @@
     w_list.convert_to_cpy_strategy(space)
     return CPyListStrategy.unerase(w_list.lstorage)
 
+def list_traverse(space, w_list, visit, args):
+    from rpython.rlib.rawrefcount import PYOBJ_HDR_PTR
+    from pypy.module.cpyext.sequence import CPyListStrategy
+    from rpython.rtyper.lltypesystem import lltype, llmemory
+    from rpython.rlib.debug import debug_print
+    assert isinstance(w_list, W_ListObject)
+    obj = llmemory.cast_ptr_to_adr(w_list.lstorage)
+    objint = llmemory.cast_adr_to_int(obj, "forced")
+    if objint != 0:
+        cpy_strategy = space.fromcache(CPyListStrategy)
+        if not cpy_strategy.locked and cpy_strategy == w_list.strategy:
+            debug_print('rrc do traverse ', w_list)
+            index = 0
+            storage = CPyListStrategy.unerase(w_list.lstorage)
+            while index < storage._length:
+                pyobj = storage._elems[index]
+                pyobj_hdr = rffi.cast(PYOBJ_HDR_PTR, pyobj)
+                if pyobj_hdr:
+                    vret = rffi.cast(lltype.Signed, visit(pyobj_hdr, args))
+                    if vret:
+                        return vret
+                index += 1
+        else:
+            debug_print('rrc no traverse', w_list, 'locked',
+                        cpy_strategy.locked, 'strategy', w_list.strategy)
+    return 0
+
 @cpython_api([rffi.VOIDP, Py_ssize_t, PyObject], lltype.Void, error=CANNOT_FAIL)
 def PyList_SET_ITEM(space, w_list, index, py_item):
     """Form of PyList_SetItem() without error checking. This is normally
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -77,9 +77,9 @@
         state = space.fromcache(State)
         return state.C._PyPy_subtype_dealloc
 
-    def get_free(self, space):
+    def get_free(self, space, gc):
         state = space.fromcache(State)
-        return state.C.PyObject_Free
+        return state.C.PyObject_GC_Del if gc else state.C.PyObject_Free
 
     def has_traverse(self, space):
         return False
@@ -156,7 +156,7 @@
                 return tp_dealloc
 
         if tp_free:
-            def get_free(self, space):
+            def get_free(self, space, gc):
                 return tp_free
 
         if tp_attach:
diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -281,6 +281,7 @@
     erase, unerase = rerased.new_erasing_pair("cpylist")
     erase = staticmethod(erase)
     unerase = staticmethod(unerase)
+    locked = False
 
     def _check_index(self, index, length):
         if index < 0:
diff --git a/pypy/module/cpyext/src/object.c b/pypy/module/cpyext/src/object.c
--- a/pypy/module/cpyext/src/object.c
+++ b/pypy/module/cpyext/src/object.c
@@ -132,10 +132,17 @@
 PyObject *
 PyType_GenericAlloc(PyTypeObject *type, Py_ssize_t nitems)
 {
+    PyObject *obj;
+
     if (PyType_IS_GC(type))
-        return (PyObject*)_PyObject_GC_NewVar(type, nitems);
+        obj = (PyObject*)_PyObject_GC_NewVar(type, nitems);
     else
-        return (PyObject*)_PyObject_NewVar(type, nitems);
+        obj = (PyObject*)_PyObject_NewVar(type, nitems);
+
+    if (PyType_IS_GC(type))
+        _PyObject_GC_TRACK(obj);
+
+    return obj;
 }
 
 PyObject *
@@ -191,7 +198,7 @@
         return (PyVarObject*)PyErr_NoMemory();
 
     if (type->tp_itemsize == 0)
-        return (PyVarObject*)PyObject_INIT(py_obj, type);
+        return PyObject_INIT(py_obj, type);
     else
         return PyObject_INIT_VAR((PyVarObject*)py_obj, type, nitems);
 }
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -133,9 +133,10 @@
                 print 'dealloc_trigger DONE'
                 return "RETRY"
             def tp_traverse(pyobj_ptr, callback, args):
-                from pypy.module.cpyext.api import PyObject
+                from pypy.module.cpyext.api import PyObject, generic_cpy_call
                 from pypy.module.cpyext.typeobjectdefs import visitproc
-                from pypy.module.cpyext.pyobject import cts
+                from pypy.module.cpyext.pyobject import cts, from_ref
+
                 # convert to pointers with correct types (PyObject)
                 callback_addr = llmemory.cast_ptr_to_adr(callback)
                 callback_ptr = llmemory.cast_adr_to_ptr(callback_addr,
@@ -148,9 +149,9 @@
                 print 'traverse PyObject', pyobj, 'of type', name
 
                 # now call tp_traverse (if possible)
-                if pyobj.c_ob_type and pyobj.c_ob_type.c_tp_traverse:
-                    pyobj.c_ob_type.c_tp_traverse(pyobj, callback_ptr,
-                                                  args)
+                if pto and pto.c_tp_traverse:
+                    generic_cpy_call(space, pto.c_tp_traverse, pyobj,
+                                     callback_ptr, args)
             rawrefcount.init(dealloc_trigger, tp_traverse)
         else:
             if space.config.translation.gc == "boehm":
@@ -176,9 +177,12 @@
                     (lambda w_obj: _clear_weakref_callbacks(w_obj))
 
                 def _tp_traverse(pyobj_ptr, callback, args):
-                    from pypy.module.cpyext.api import PyObject
+                    from pypy.module.cpyext.api import PyObject, \
+                        generic_cpy_call
                     from pypy.module.cpyext.typeobjectdefs import visitproc
-                    from pypy.module.cpyext.pyobject import cts
+                    from pypy.module.cpyext.pyobject import cts, from_ref
+                    from pypy.module.cpyext.listobject import list_traverse
+
                     # convert to pointers with correct types (PyObject)
                     callback_addr = llmemory.cast_ptr_to_adr(callback)
                     callback_ptr = llmemory.cast_adr_to_ptr(callback_addr,
@@ -208,9 +212,19 @@
                                 debug_print(" " * i * 3, "unknown base")
                             pto2 = base
                             i += 1
+
+                    if pyobj.c_ob_pypy_link != 0: # special traverse
+                        w_obj = from_ref(space, pyobj)
+                        w_obj_type = space.type(w_obj)
+                        if space.is_w(w_obj_type, space.w_list): # list
+                            debug_print('rrc list traverse ', pyobj)
+                            list_traverse(space, w_obj, callback, args)
+                            return
+
                     if pto and pto.c_tp_traverse:
                         debug_print("rrc do traverse", pyobj)
-                        pto.c_tp_traverse(pyobj, callback_ptr, args)
+                        generic_cpy_call(space, pto.c_tp_traverse, pyobj,
+                                         callback_ptr, args)
 
                 self.tp_traverse = (lambda o, v, a:_tp_traverse(o, v, a))
 
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -434,7 +434,7 @@
         pto.c_tp_flags |= Py_TPFLAGS_UNICODE_SUBCLASS
     elif space.issubtype_w(w_obj, space.w_tuple):
         pto.c_tp_flags |= Py_TPFLAGS_TUPLE_SUBCLASS
-        pto.c_tp_flags |= Py_TPFLAGS_HAVE_GC
+        pto.c_tp_flags |= Py_TPFLAGS_HAVE_GC # TODO: remove? already done elsewhere
     elif space.issubtype_w(w_obj, space.w_list):
         pto.c_tp_flags |= Py_TPFLAGS_LIST_SUBCLASS
     elif space.issubtype_w(w_obj, space.w_dict):
@@ -610,6 +610,8 @@
     elif space.is_w(w_type, space.w_tuple):
         pto.c_tp_itemsize = rffi.sizeof(PyObject)
         pto.c_tp_flags |= Py_TPFLAGS_HAVE_GC
+    elif space.is_w(w_type, space.w_list):
+        pto.c_tp_flags |= Py_TPFLAGS_HAVE_GC
 
     state = space.fromcache(State)
     pto.c_tp_alloc = state.C.PyType_GenericAlloc
@@ -646,13 +648,6 @@
             # strange, but happens (ABCMeta)
             pto.c_tp_dealloc = state.C._PyPy_subtype_dealloc
 
-    # free
-    if space.gettypeobject(w_type.layout.typedef) is w_type:
-        # only for the exact type, like 'space.w_tuple' or 'space.w_list'
-        pto.c_tp_free = typedescr.get_free(space)
-    else:
-        pto.c_tp_free = pto.c_tp_base.c_tp_free
-
     from rpython.rtyper.lltypesystem import llmemory
     from pypy.module.cpyext.typeobjectdefs import traverseproc
     if typedescr.has_traverse(space):
@@ -685,6 +680,18 @@
         if pto.c_tp_base != base_object_pto or flags & Py_TPFLAGS_HEAPTYPE:
                 pto.c_tp_new = pto.c_tp_base.c_tp_new
         decref(space, base_object_pyo)
+
+    # free
+    gc = pto.c_tp_flags & Py_TPFLAGS_HAVE_GC != 0
+    if space.gettypeobject(w_type.layout.typedef) is w_type:
+        # only for the exact type, like 'space.w_tuple' or 'space.w_list'
+        pto.c_tp_free = typedescr.get_free(space, gc)
+    else:
+        if gc:
+            pto.c_tp_free = state.C.PyObject_GC_Del
+        else:
+            pto.c_tp_free = state.C.PyObject_Free
+
     pto.c_tp_flags |= Py_TPFLAGS_READY
     return pto
 
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -248,9 +248,11 @@
         cpy_strategy = self.space.fromcache(CPyListStrategy)
         if self.strategy is cpy_strategy:
             return
+        cpy_strategy.locked = True
         lst = self.getitems()
         self.strategy = cpy_strategy
         self.lstorage = cpy_strategy.erase(CPyListStorage(space, lst))
+        cpy_strategy.locked = False
 
     # ___________________________________________________
 
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -3383,6 +3383,14 @@
                 #pygchdr = self.rrc_pyobj_as_gc(self._pyobj(pyobject))
                 #if pygchdr <> lltype.nullptr(self.PYOBJ_GC_HDR):
                 #    self._rrc_debug_check_list(pygchdr)
+
+                # TODO?
+                #pygchdr = self.rrc_pyobj_as_gc(self._pyobj(pyobject))
+                #if pygchdr <> lltype.nullptr(self.PYOBJ_GC_HDR):
+                #    next = pygchdr.c_gc_next
+                #    next.c_gc_prev = pygchdr.c_gc_prev
+                #    pygchdr.c_gc_prev.c_gc_next = next
+
                 self.rrc_dealloc_pending.append(pyobject)
                 # an object with refcnt == 0 cannot stay around waiting
                 # for its deallocator to be called.  Some code (lxml)
@@ -3434,7 +3442,7 @@
 
         # TODO: check again, if some new border objects have been marked and
         #       continue marking recursively... why needed? -> wrapper for
-        #       pypy-obj is no pygc-obj??? ...
+        #       pypy-obj is no pygc-obj??? ...KI
 
         self._rrc_debug_check_consistency(print_label="end-mark")
         #self.rrc_o_list_old.foreach(self._rrc_major_trace, use_cylicrc) # TODO: remove?
@@ -3470,6 +3478,14 @@
             self.visit_all_objects()
 
     def rrc_major_collection_free(self):
+        if self.rrc_state == self.RAWREFCOUNT_STATE_DEFAULT:
+            if not self._rrc_gc_list_is_empty(self.rrc_pyobj_old_list):
+                self._rrc_debug_check_consistency()
+                self._rrc_clear_weakref_callbacks()
+                self._rrc_gc_list_merge(self.rrc_pyobj_old_list,
+                                        self.rrc_pyobj_dead_list)
+                self._rrc_debug_check_consistency(print_label="before-sweep")
+
         ll_assert(self.rrc_p_dict_nurs.length() == 0, "p_dict_nurs not empty 2")
         length_estimate = self.rrc_p_dict.length()
         self.rrc_p_dict.delete()
@@ -3489,14 +3505,6 @@
         self.rrc_o_list_old.delete()
         self.rrc_o_list_old = new_o_list
 
-        if self.rrc_state == self.RAWREFCOUNT_STATE_DEFAULT:
-            if not self._rrc_gc_list_is_empty(self.rrc_pyobj_old_list):
-                self._rrc_debug_check_consistency()
-                self._rrc_clear_weakref_callbacks()
-                self._rrc_gc_list_merge(self.rrc_pyobj_old_list,
-                                        self.rrc_pyobj_dead_list)
-                self._rrc_debug_check_consistency(print_label="before-sweep")
-
     def _rrc_clear_weakref_callbacks(self):
         # Look for any weakrefs within the trash cycle and remove the callback.
         # This is only needed for weakrefs created from rawrefcounted objects
@@ -3564,13 +3572,13 @@
             self._rrc_pyobj_gc_refcnt_set(pygchdr, refcnt)
             pygchdr = pygchdr.c_gc_next
 
-        self._rrc_debug_check_consistency(print_label="rc-initialized")
-
         # For every object in this set, if it is marked, add 1 as a real
         # refcount (p_list => pyobj stays alive if obj stays alive).
         self.rrc_p_list_old.foreach(self._rrc_obj_fix_refcnt, None)
         self.rrc_o_list_old.foreach(self._rrc_obj_fix_refcnt, None)
 
+        self._rrc_debug_check_consistency(print_label="rc-initialized")
+
         # Subtract all internal refcounts from the cyclic refcount
         # of rawrefcounted objects
         pygchdr = pygclist.c_gc_next

From pypy.commits at gmail.com  Fri May 31 15:40:06 2019
From: pypy.commits at gmail.com (mattip)
Date: Fri, 31 May 2019 12:40:06 -0700 (PDT)
Subject: [pypy-commit] pypy win32-encoding: merge py3.6 into branch
Message-ID: <5cf18316.1c69fb81.3c43a.7180@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-encoding
Changeset: r96724:b5fb07545a94
Date: 2019-05-31 22:39 +0300
http://bitbucket.org/pypy/pypy/changeset/b5fb07545a94/

Log:	merge py3.6 into branch

diff --git a/pypy/module/posix/interp_nt.py b/pypy/module/posix/interp_nt.py
--- a/pypy/module/posix/interp_nt.py
+++ b/pypy/module/posix/interp_nt.py
@@ -1,5 +1,6 @@
 from rpython.rlib import rwin32
 from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rwin32file import make_win32_traits
 from rpython.rlib._os_support import UnicodeTraits
 from rpython.translator import cdir
@@ -91,6 +92,7 @@
     assert traits.str is unicode, 'Currently only handles unicode paths'
     win32traits = make_traits(traits)
 
+    @specialize.argtype(0)
     def _getfinalpathname_llimpl(path):
         if not win32traits.check_GetFinalPathNameByHandle():
             raise LLNotImplemented("GetFinalPathNameByHandle not available on "
@@ -122,7 +124,8 @@
                     VOLUME_NAME_DOS)
                 if result == 0:
                     raise rwin32.lastSavedWindowsError("GetFinalPathNameByHandle")
-                return buf.str(rffi.cast(lltype.Signed, result))
+                res = buf.str(rffi.cast(lltype.Signed, result))
+                return res.encode('utf8'), len(res)
         finally:
             rwin32.CloseHandle(hFile)
 
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -2277,15 +2277,14 @@
                                space.newint(info[2])])
 
     def _getfinalpathname(space, w_path):
-        path = space.utf8_w(w_path).decode('utf-8')
         try:
-            result = nt._getfinalpathname(path)
+            s, lgt = dispatch_filename(nt._getfinalpathname)(space, w_path)
         except nt.LLNotImplemented as e:
             raise OperationError(space.w_NotImplementedError,
                                  space.newtext(e.msg))
         except OSError as e:
             raise wrap_oserror2(space, e, w_path, eintr_retry=False)
-        return space.newtext(result)
+        return space.newtext(s, lgt)
 
 
 def chflags():
diff --git a/pypy/module/posix/test/test_nt.py b/pypy/module/posix/test/test_nt.py
--- a/pypy/module/posix/test/test_nt.py
+++ b/pypy/module/posix/test/test_nt.py
@@ -19,9 +19,9 @@
 
 
 def test__getfinalpathname():
-    path = __file__.decode('mbcs')
+    path = __file__.decode('utf-8')
     try:
-        result = nt._getfinalpathname(path)
+        result, lgt = nt._getfinalpathname(path)
     except nt.LLNotImplemented:
         py.test.skip("_getfinalpathname not supported on this platform")
     assert os.path.exists(result)
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -383,7 +383,7 @@
     def newbytearray(self, l):
         return W_BytearrayObject(l)
 
-    # XXX TODO - remove this and force all users to call with utf8
+    # XXX TODO - remove the specialization and force all users to call with utf8
     @specialize.argtype(1)
     def newtext(self, s, lgt=-1, unused=-1):
         # the unused argument can be from something like