From pypy.commits at gmail.com  Fri Nov  1 13:00:10 2019
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 01 Nov 2019 10:00:10 -0700 (PDT)
Subject: [pypy-commit] pypy py3.7: fix bad merge
Message-ID: <5dbc649a.1c69fb81.5af5b.1c96@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.7
Changeset: r97926:efacfd8fec99
Date: 2019-11-01 16:59 +0000
http://bitbucket.org/pypy/pypy/changeset/efacfd8fec99/

Log:	fix bad merge

diff --git a/lib-python/3/test/test_capi.py b/lib-python/3/test/test_capi.py
--- a/lib-python/3/test/test_capi.py
+++ b/lib-python/3/test/test_capi.py
@@ -296,8 +296,6 @@
                 return tuple(super().items())
         dict_obj = {'foo': 1, 'bar': 2, 'spam': 3}
 
- at unittest.skipIf(support.check_impl_detail(pypy=True),
-                 'Py_AddPendingCall not currently supported.')
         for mapping in [{}, OrderedDict(), Mapping1(), Mapping2(),
                         dict_obj, OrderedDict(dict_obj),
                         Mapping1(dict_obj), Mapping2(dict_obj)]:
@@ -326,6 +324,8 @@
         self.assertRaises(TypeError, _testcapi.get_mapping_items, bad_mapping)
 
 
+ at unittest.skipIf(support.check_impl_detail(pypy=True),
+                 'Py_AddPendingCall not currently supported.')
 class TestPendingCalls(unittest.TestCase):
 
     def pendingcalls_submit(self, l, n):

From pypy.commits at gmail.com  Fri Nov  1 17:14:49 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 01 Nov 2019 14:14:49 -0700 (PDT)
Subject: [pypy-commit] pypy expose-jsonmap: merge default
Message-ID: <5dbca049.1c69fb81.fe9a7.c5e9@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: expose-jsonmap
Changeset: r97927:bae753faf6ac
Date: 2019-11-01 22:01 +0100
http://bitbucket.org/pypy/pypy/changeset/bae753faf6ac/

Log:	merge default

diff too long, truncating to 2000 out of 7524 lines

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -3,10 +3,11 @@
 License
 =======
 
-Except when otherwise stated (look for LICENSE files in directories or
-information at the beginning of each file) all software and documentation in
-the 'rpython', 'pypy', 'ctype_configure', 'dotviewer', 'demo', 'lib_pypy',
-'py', and '_pytest' directories is licensed as follows:
+Except when otherwise stated (look for LICENSE files in directories
+or information at the beginning of each file) all software and
+documentation in the 'rpython', 'pypy', 'ctype_configure', 'dotviewer',
+'demo', 'extra_tests', 'include', 'lib_pypy', 'py', and '_pytest'
+directories is licensed as follows:
 
     The MIT License
 
diff --git a/README.rst b/README.rst
--- a/README.rst
+++ b/README.rst
@@ -9,15 +9,15 @@
 
 The home page for the interpreter is:
 
-    http://pypy.org/
+    https://pypy.org/
 
 If you want to help developing PyPy, this documentation might help you:
 
-    http://doc.pypy.org/
+    https://doc.pypy.org/
 
 More documentation about the RPython framework can be found here:
 
-    http://rpython.readthedocs.io/
+    https://rpython.readthedocs.io/
 
 The source for the documentation is in the pypy/doc directory.
 
@@ -25,7 +25,7 @@
 Using PyPy instead of CPython
 -----------------------------
 
-Please read the information at http://pypy.org/ to find the correct way to
+Please read the information at https://pypy.org/ to find the correct way to
 download and use PyPy as an alternative to CPython. 
 
 
@@ -36,7 +36,7 @@
 interpreter. It is time-consuming and requires significant computing resources.
 More information can be found here:
 
-    http://doc.pypy.org/en/latest/build.html
+    https://doc.pypy.org/en/latest/build.html
 
 Enjoy and send us feedback!
 
diff --git a/extra_tests/cffi_tests/cffi0/test_function.py b/extra_tests/cffi_tests/cffi0/test_function.py
--- a/extra_tests/cffi_tests/cffi0/test_function.py
+++ b/extra_tests/cffi_tests/cffi0/test_function.py
@@ -114,7 +114,7 @@
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
             int fputs(const char *, void *);
-            void *stderr;
+            extern void *stderr;
         """)
         needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
@@ -131,7 +131,7 @@
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
             int fputs(char *, void *);
-            void *stderr;
+            extern void *stderr;
         """)
         needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
@@ -148,7 +148,7 @@
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
            int fprintf(void *, const char *format, ...);
-           void *stderr;
+           extern void *stderr;
         """)
         needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
@@ -210,7 +210,7 @@
             py.test.skip("probably no symbol 'stderr' in the lib")
         ffi.cdef("""
             int fputs(const char *, void *);
-            void *stderr;
+            extern void *stderr;
         """)
         needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
@@ -257,7 +257,7 @@
             py.test.skip("probably no symbol 'stdout' in the lib")
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
-            void *stdout;
+            extern void *stdout;
         """)
         needs_dlopen_none()
         C = ffi.dlopen(None)
@@ -497,7 +497,7 @@
         ffi.cdef("""
             typedef enum { MYE1, MYE2 } myenum_t;
             double myfunc(double);
-            double myvar;
+            extern double myvar;
             const double myconst;
             #define MYFOO 42
         """)
@@ -508,7 +508,7 @@
         if self.Backend is CTypesBackend:
             py.test.skip("not with the ctypes backend")
         ffi = FFI(backend=self.Backend())
-        ffi.cdef("int foobar(void); int foobaz;")
+        ffi.cdef("int foobar(void); extern int foobaz;")
         lib = ffi.dlopen(lib_m)
         ffi.dlclose(lib)
         e = py.test.raises(ValueError, getattr, lib, 'foobar')
diff --git a/extra_tests/cffi_tests/cffi0/test_ownlib.py b/extra_tests/cffi_tests/cffi0/test_ownlib.py
--- a/extra_tests/cffi_tests/cffi0/test_ownlib.py
+++ b/extra_tests/cffi_tests/cffi0/test_ownlib.py
@@ -202,7 +202,7 @@
             py.test.skip("fix the auto-generation of the tiny test lib")
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
-            int my_array[7];
+            extern int my_array[7];
         """)
         ownlib = ffi.dlopen(self.module)
         for i in range(7):
@@ -224,7 +224,7 @@
             py.test.skip("not supported by the ctypes backend")
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
-            int my_array[];
+            extern int my_array[];
         """)
         ownlib = ffi.dlopen(self.module)
         for i in range(7):
@@ -292,7 +292,7 @@
                 long bottom;
             } RECT;
             
-            long left, top, right, bottom;
+            extern long left, top, right, bottom;
 
             RECT ReturnRect(int i, RECT ar, RECT* br, POINT cp, RECT dr,
                         RECT *er, POINT fp, RECT gr);
@@ -322,7 +322,7 @@
         if self.Backend is CTypesBackend:
             py.test.skip("not implemented with the ctypes backend")
         ffi = FFI(backend=self.Backend())
-        ffi.cdef("long left; int test_getting_errno(void);")
+        ffi.cdef("extern long left; int test_getting_errno(void);")
         lib = ffi.dlopen(self.module)
         lib.left = 123456
         p = ffi.addressof(lib, "left")
diff --git a/extra_tests/cffi_tests/cffi0/test_parsing.py b/extra_tests/cffi_tests/cffi0/test_parsing.py
--- a/extra_tests/cffi_tests/cffi0/test_parsing.py
+++ b/extra_tests/cffi_tests/cffi0/test_parsing.py
@@ -325,6 +325,7 @@
     assert value == sys.maxsize * 2 - 40
 
 def test__is_constant_globalvar():
+    import warnings
     for input, expected_output in [
         ("int a;",          False),
         ("const int a;",    True),
@@ -342,10 +343,13 @@
         ("const int a[5][6];", False),
         ]:
         ffi = FFI()
-        ffi.cdef(input)
+        with warnings.catch_warnings(record=True) as log:
+            warnings.simplefilter("always")
+            ffi.cdef(input)
         declarations = ffi._parser._declarations
         assert ('constant a' in declarations) == expected_output
         assert ('variable a' in declarations) == (not expected_output)
+        assert len(log) == (1 - expected_output)
 
 def test_restrict():
     from cffi import model
@@ -355,7 +359,7 @@
         ("int *a;",            False),
         ]:
         ffi = FFI()
-        ffi.cdef(input)
+        ffi.cdef("extern " + input)
         tp, quals = ffi._parser._declarations['variable a']
         assert bool(quals & model.Q_RESTRICT) == expected_output
 
diff --git a/extra_tests/cffi_tests/cffi0/test_verify.py b/extra_tests/cffi_tests/cffi0/test_verify.py
--- a/extra_tests/cffi_tests/cffi0/test_verify.py
+++ b/extra_tests/cffi_tests/cffi0/test_verify.py
@@ -287,7 +287,7 @@
 def test_var_signed_integer_types():
     ffi = FFI()
     lst = all_signed_integer_types(ffi)
-    csource = "\n".join(["%s somevar_%s;" % (tp, tp.replace(' ', '_'))
+    csource = "\n".join(["static %s somevar_%s;" % (tp, tp.replace(' ', '_'))
                          for tp in lst])
     ffi.cdef(csource)
     lib = ffi.verify(csource)
@@ -306,7 +306,7 @@
 def test_var_unsigned_integer_types():
     ffi = FFI()
     lst = all_unsigned_integer_types(ffi)
-    csource = "\n".join(["%s somevar_%s;" % (tp, tp.replace(' ', '_'))
+    csource = "\n".join(["static %s somevar_%s;" % (tp, tp.replace(' ', '_'))
                          for tp in lst])
     ffi.cdef(csource)
     lib = ffi.verify(csource)
@@ -818,8 +818,8 @@
 
 def test_access_variable():
     ffi = FFI()
-    ffi.cdef("int foo(void);\n"
-             "int somenumber;")
+    ffi.cdef("static int foo(void);\n"
+             "static int somenumber;")
     lib = ffi.verify("""
         static int somenumber = 2;
         static int foo(void) {
@@ -836,7 +836,7 @@
 def test_access_address_of_variable():
     # access the address of 'somenumber': need a trick
     ffi = FFI()
-    ffi.cdef("int somenumber; static int *const somenumberptr;")
+    ffi.cdef("static int somenumber; static int *const somenumberptr;")
     lib = ffi.verify("""
         static int somenumber = 2;
         #define somenumberptr (&somenumber)
@@ -849,7 +849,7 @@
 def test_access_array_variable(length=5):
     ffi = FFI()
     ffi.cdef("int foo(int);\n"
-             "int somenumber[%s];" % (length,))
+             "static int somenumber[%s];" % (length,))
     lib = ffi.verify("""
         static int somenumber[] = {2, 2, 3, 4, 5};
         static int foo(int i) {
@@ -881,7 +881,7 @@
     ffi = FFI()
     ffi.cdef("struct foo { int x; ...; };\n"
              "int foo(int);\n"
-             "struct foo stuff;")
+             "static struct foo stuff;")
     lib = ffi.verify("""
         struct foo { int x, y, z; };
         static struct foo stuff = {2, 5, 8};
@@ -905,9 +905,9 @@
 
 def test_access_callback():
     ffi = FFI()
-    ffi.cdef("int (*cb)(int);\n"
-             "int foo(int);\n"
-             "void reset_cb(void);")
+    ffi.cdef("static int (*cb)(int);\n"
+             "static int foo(int);\n"
+             "static void reset_cb(void);")
     lib = ffi.verify("""
         static int g(int x) { return x * 7; }
         static int (*cb)(int);
@@ -923,9 +923,9 @@
 def test_access_callback_function_typedef():
     ffi = FFI()
     ffi.cdef("typedef int mycallback_t(int);\n"
-             "mycallback_t *cb;\n"
-             "int foo(int);\n"
-             "void reset_cb(void);")
+             "static mycallback_t *cb;\n"
+             "static int foo(int);\n"
+             "static void reset_cb(void);")
     lib = ffi.verify("""
         static int g(int x) { return x * 7; }
         static int (*cb)(int);
@@ -1075,7 +1075,7 @@
 def test_autofilled_struct_as_argument_dynamic():
     ffi = FFI()
     ffi.cdef("struct foo_s { long a; ...; };\n"
-             "int (*foo)(struct foo_s);")
+             "static int (*foo)(struct foo_s);")
     lib = ffi.verify("""
         struct foo_s {
             double b;
@@ -1084,7 +1084,7 @@
         int foo1(struct foo_s s) {
             return (int)s.a - (int)s.b;
         }
-        int (*foo)(struct foo_s s) = &foo1;
+        static int (*foo)(struct foo_s s) = &foo1;
     """)
     e = py.test.raises(NotImplementedError, lib.foo, "?")
     msg = ("ctype 'struct foo_s' not supported as argument.  It is a struct "
@@ -1454,7 +1454,7 @@
         py.test.skip("_Bool not in MSVC")
     ffi = FFI()
     ffi.cdef("struct foo_s { _Bool x; };"
-             "_Bool foo(_Bool); _Bool (*foop)(_Bool);")
+             "_Bool foo(_Bool); static _Bool (*foop)(_Bool);")
     lib = ffi.verify("""
         struct foo_s { _Bool x; };
         int foo(int arg) {
@@ -1463,7 +1463,7 @@
         _Bool _foofunc(_Bool x) {
             return !x;
         }
-        _Bool (*foop)(_Bool) = _foofunc;
+        static _Bool (*foop)(_Bool) = _foofunc;
     """)
     p = ffi.new("struct foo_s *")
     p.x = 1
@@ -1654,7 +1654,7 @@
 
 def test_FILE_stored_explicitly():
     ffi = FFI()
-    ffi.cdef("int myprintf11(const char *, int); FILE *myfile;")
+    ffi.cdef("int myprintf11(const char *, int); extern FILE *myfile;")
     lib = ffi.verify("""
         #include <stdio.h>
         FILE *myfile;
@@ -1680,19 +1680,19 @@
 
 def test_global_array_with_missing_length():
     ffi = FFI()
-    ffi.cdef("int fooarray[];")
+    ffi.cdef("extern int fooarray[];")
     lib = ffi.verify("int fooarray[50];")
     assert repr(lib.fooarray).startswith("<cdata 'int *'")
 
 def test_global_array_with_dotdotdot_length():
     ffi = FFI()
-    ffi.cdef("int fooarray[...];")
+    ffi.cdef("extern int fooarray[...];")
     lib = ffi.verify("int fooarray[50];")
     assert repr(lib.fooarray).startswith("<cdata 'int[50]'")
 
 def test_bad_global_array_with_dotdotdot_length():
     ffi = FFI()
-    ffi.cdef("int fooarray[...];")
+    ffi.cdef("extern int fooarray[...];")
     py.test.raises(VerificationError, ffi.verify, "char fooarray[23];")
 
 def test_struct_containing_struct():
@@ -1813,7 +1813,7 @@
 def test_callback_indirection():
     ffi = FFI()
     ffi.cdef("""
-        int (*python_callback)(int how_many, int *values);
+        static int (*python_callback)(int how_many, int *values);
         int (*const c_callback)(int,...);   /* pass this ptr to C routines */
         int some_c_function(int(*cb)(int,...));
     """)
@@ -1947,24 +1947,24 @@
 
 def test_bug_const_char_ptr_array_1():
     ffi = FFI()
-    ffi.cdef("""const char *a[...];""")
+    ffi.cdef("""extern const char *a[...];""")
     lib = ffi.verify("""const char *a[5];""")
     assert repr(ffi.typeof(lib.a)) == "<ctype 'char *[5]'>"
 
 def test_bug_const_char_ptr_array_2():
     from cffi import FFI     # ignore warnings
     ffi = FFI()
-    ffi.cdef("""const int a[];""")
+    ffi.cdef("""extern const int a[];""")
     lib = ffi.verify("""const int a[5];""")
     assert repr(ffi.typeof(lib.a)) == "<ctype 'int *'>"
 
 def _test_various_calls(force_libffi):
     cdef_source = """
-    int xvalue;
-    long long ivalue, rvalue;
-    float fvalue;
-    double dvalue;
-    long double Dvalue;
+    extern int xvalue;
+    extern long long ivalue, rvalue;
+    extern float fvalue;
+    extern double dvalue;
+    extern long double Dvalue;
     signed char tf_bb(signed char x, signed char c);
     unsigned char tf_bB(signed char x, unsigned char c);
     short tf_bh(signed char x, short c);
@@ -2148,7 +2148,7 @@
     # exported symbols as well.  So we must not export a simple name
     # like 'foo'!
     ffi1 = FFI()
-    ffi1.cdef("int foo_verify_dlopen_flags;")
+    ffi1.cdef("extern int foo_verify_dlopen_flags;")
 
     lib1 = ffi1.verify("int foo_verify_dlopen_flags;",
                        flags=ffi1.RTLD_GLOBAL | ffi1.RTLD_LAZY)
@@ -2162,7 +2162,7 @@
 def get_second_lib():
     # Hack, using modulename makes the test fail
     ffi2 = FFI()
-    ffi2.cdef("int foo_verify_dlopen_flags;")
+    ffi2.cdef("extern int foo_verify_dlopen_flags;")
     lib2 = ffi2.verify("int foo_verify_dlopen_flags;",
                        flags=ffi2.RTLD_GLOBAL | ffi2.RTLD_LAZY)
     return lib2
diff --git a/extra_tests/cffi_tests/cffi1/test_dlopen.py b/extra_tests/cffi_tests/cffi1/test_dlopen.py
--- a/extra_tests/cffi_tests/cffi1/test_dlopen.py
+++ b/extra_tests/cffi_tests/cffi1/test_dlopen.py
@@ -7,7 +7,7 @@
 
 def test_simple():
     ffi = FFI()
-    ffi.cdef("int close(int); static const int BB = 42; int somevar;")
+    ffi.cdef("int close(int); static const int BB = 42; extern int somevar;")
     target = udir.join('test_simple.py')
     make_py_source(ffi, 'test_simple', str(target))
     assert target.read() == r"""# auto-generated file
@@ -197,7 +197,7 @@
 
 def test_global_var():
     ffi = FFI()
-    ffi.cdef("int myglob;")
+    ffi.cdef("extern int myglob;")
     target = udir.join('test_global_var.py')
     make_py_source(ffi, 'test_global_var', str(target))
     assert target.read() == r"""# auto-generated file
diff --git a/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py
--- a/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py
+++ b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py
@@ -1780,7 +1780,7 @@
 
     def test_import_from_lib(self):
         ffi2 = cffi.FFI()
-        ffi2.cdef("int myfunc(int); int myvar;\n#define MYFOO ...\n")
+        ffi2.cdef("int myfunc(int); extern int myvar;\n#define MYFOO ...\n")
         outputfilename = recompile(ffi2, "_test_import_from_lib",
                                    "int myfunc(int x) { return x + 1; }\n"
                                    "int myvar = -5;\n"
diff --git a/extra_tests/cffi_tests/cffi1/test_re_python.py b/extra_tests/cffi_tests/cffi1/test_re_python.py
--- a/extra_tests/cffi_tests/cffi1/test_re_python.py
+++ b/extra_tests/cffi_tests/cffi1/test_re_python.py
@@ -64,11 +64,11 @@
     #define BIGNEG -420000000000L
     int add42(int);
     int add43(int, ...);
-    int globalvar42;
+    extern int globalvar42;
     const int globalconst42;
     const char *const globalconsthello;
     int no_such_function(int);
-    int no_such_globalvar;
+    extern int no_such_globalvar;
     struct foo_s;
     typedef struct bar_s { int x; signed char a[]; } bar_t;
     enum foo_e { AA, BB, CC };
@@ -76,6 +76,7 @@
     struct with_union { union { int a; char b; }; };
     union with_struct { struct { int a; char b; }; };
     struct NVGcolor { union { float rgba[4]; struct { float r,g,b,a; }; }; };
+    typedef struct selfref { struct selfref *next; } *selfref_ptr_t;
     """)
     ffi.set_source('re_python_pysrc', None)
     ffi.emit_python_code(str(tmpdir.join('re_python_pysrc.py')))
@@ -255,3 +256,8 @@
     assert ffi.offsetof("struct NVGcolor", "g") == FLOAT
     assert ffi.offsetof("struct NVGcolor", "b") == FLOAT * 2
     assert ffi.offsetof("struct NVGcolor", "a") == FLOAT * 3
+
+def test_selfref():
+    # based on issue #429
+    from re_python_pysrc import ffi
+    ffi.new("selfref_ptr_t")
diff --git a/extra_tests/cffi_tests/cffi1/test_recompiler.py b/extra_tests/cffi_tests/cffi1/test_recompiler.py
--- a/extra_tests/cffi_tests/cffi1/test_recompiler.py
+++ b/extra_tests/cffi_tests/cffi1/test_recompiler.py
@@ -84,7 +84,7 @@
                      "(FUNCTION 1)(PRIMITIVE 7)(FUNCTION_END 1)(POINTER 0)")
 
 def test_type_table_array():
-    check_type_table("int a[100];",
+    check_type_table("extern int a[100];",
                      "(PRIMITIVE 7)(ARRAY 0)(None 100)")
 
 def test_type_table_typedef():
@@ -159,7 +159,7 @@
 
 def test_global_var_array():
     ffi = FFI()
-    ffi.cdef("int a[100];")
+    ffi.cdef("extern int a[100];")
     lib = verify(ffi, 'test_global_var_array', 'int a[100] = { 9999 };')
     lib.a[42] = 123456
     assert lib.a[42] == 123456
@@ -183,7 +183,7 @@
 
 def test_global_var_int():
     ffi = FFI()
-    ffi.cdef("int a, b, c;")
+    ffi.cdef("extern int a, b, c;")
     lib = verify(ffi, 'test_global_var_int', 'int a = 999, b, c;')
     assert lib.a == 999
     lib.a -= 1001
@@ -284,7 +284,7 @@
 
 def test_dir():
     ffi = FFI()
-    ffi.cdef("int ff(int); int aa; static const int my_constant;")
+    ffi.cdef("int ff(int); extern int aa; static const int my_constant;")
     lib = verify(ffi, 'test_dir', """
         #define my_constant  (-45)
         int aa;
@@ -406,7 +406,7 @@
 
 def test_dotdotdot_global_array():
     ffi = FFI()
-    ffi.cdef("int aa[...]; int bb[...];")
+    ffi.cdef("extern int aa[...]; extern int bb[...];")
     lib = verify(ffi, 'test_dotdotdot_global_array',
                  "int aa[41]; int bb[12];")
     assert ffi.sizeof(lib.aa) == 41 * 4
@@ -561,37 +561,37 @@
 
 def test_bad_size_of_global_1():
     ffi = FFI()
-    ffi.cdef("short glob;")
+    ffi.cdef("extern short glob;")
     py.test.raises(VerificationError, verify, ffi,
                    "test_bad_size_of_global_1", "long glob;")
 
 def test_bad_size_of_global_2():
     ffi = FFI()
-    ffi.cdef("int glob[10];")
+    ffi.cdef("extern int glob[10];")
     py.test.raises(VerificationError, verify, ffi,
                    "test_bad_size_of_global_2", "int glob[9];")
 
 def test_unspecified_size_of_global_1():
     ffi = FFI()
-    ffi.cdef("int glob[];")
+    ffi.cdef("extern int glob[];")
     lib = verify(ffi, "test_unspecified_size_of_global_1", "int glob[10];")
     assert ffi.typeof(lib.glob) == ffi.typeof("int *")
 
 def test_unspecified_size_of_global_2():
     ffi = FFI()
-    ffi.cdef("int glob[][5];")
+    ffi.cdef("extern int glob[][5];")
     lib = verify(ffi, "test_unspecified_size_of_global_2", "int glob[10][5];")
     assert ffi.typeof(lib.glob) == ffi.typeof("int(*)[5]")
 
 def test_unspecified_size_of_global_3():
     ffi = FFI()
-    ffi.cdef("int glob[][...];")
+    ffi.cdef("extern int glob[][...];")
     lib = verify(ffi, "test_unspecified_size_of_global_3", "int glob[10][5];")
     assert ffi.typeof(lib.glob) == ffi.typeof("int(*)[5]")
 
 def test_unspecified_size_of_global_4():
     ffi = FFI()
-    ffi.cdef("int glob[...][...];")
+    ffi.cdef("extern int glob[...][...];")
     lib = verify(ffi, "test_unspecified_size_of_global_4", "int glob[10][5];")
     assert ffi.typeof(lib.glob) == ffi.typeof("int[10][5]")
 
@@ -814,7 +814,7 @@
 def test_address_of_global_var():
     ffi = FFI()
     ffi.cdef("""
-        long bottom, bottoms[2];
+        extern long bottom, bottoms[2];
         long FetchRectBottom(void);
         long FetchRectBottoms1(void);
         #define FOOBAR 42
@@ -969,7 +969,7 @@
     ffi = FFI()
     ffi.cdef("""
         typedef ... opaque_t;
-        opaque_t globvar;
+        extern opaque_t globvar;
     """)
     lib = verify(ffi, 'test_variable_of_unknown_size', """
         typedef char opaque_t[6];
@@ -1014,7 +1014,7 @@
 def test_call_with_incomplete_structs():
     ffi = FFI()
     ffi.cdef("typedef struct {...;} foo_t; "
-             "foo_t myglob; "
+             "extern foo_t myglob; "
              "foo_t increment(foo_t s); "
              "double getx(foo_t s);")
     lib = verify(ffi, 'test_call_with_incomplete_structs', """
@@ -1058,7 +1058,7 @@
 
 def test_global_var_array_2():
     ffi = FFI()
-    ffi.cdef("int a[...][...];")
+    ffi.cdef("extern int a[...][...];")
     lib = verify(ffi, 'test_global_var_array_2', 'int a[10][8];')
     lib.a[9][7] = 123456
     assert lib.a[9][7] == 123456
@@ -1071,7 +1071,7 @@
 
 def test_global_var_array_3():
     ffi = FFI()
-    ffi.cdef("int a[][...];")
+    ffi.cdef("extern int a[][...];")
     lib = verify(ffi, 'test_global_var_array_3', 'int a[10][8];')
     lib.a[9][7] = 123456
     assert lib.a[9][7] == 123456
@@ -1082,7 +1082,7 @@
 
 def test_global_var_array_4():
     ffi = FFI()
-    ffi.cdef("int a[10][...];")
+    ffi.cdef("extern int a[10][...];")
     lib = verify(ffi, 'test_global_var_array_4', 'int a[10][8];')
     lib.a[9][7] = 123456
     assert lib.a[9][7] == 123456
@@ -1205,7 +1205,7 @@
 
 def test_import_from_lib():
     ffi = FFI()
-    ffi.cdef("int mybar(int); int myvar;\n#define MYFOO ...")
+    ffi.cdef("int mybar(int); static int myvar;\n#define MYFOO ...")
     lib = verify(ffi, 'test_import_from_lib',
                  "#define MYFOO 42\n"
                  "static int mybar(int x) { return x + 1; }\n"
@@ -1221,7 +1221,7 @@
 
 def test_macro_var_callback():
     ffi = FFI()
-    ffi.cdef("int my_value; int *(*get_my_value)(void);")
+    ffi.cdef("extern int my_value; extern int *(*get_my_value)(void);")
     lib = verify(ffi, 'test_macro_var_callback',
                  "int *(*get_my_value)(void);\n"
                  "#define my_value (*get_my_value())")
@@ -1336,7 +1336,7 @@
 
 def test_const_function_type_args():
     ffi = FFI()
-    ffi.cdef("""int (*foobar)(const int a, const int *b, const int c[]);""")
+    ffi.cdef("""extern int(*foobar)(const int a,const int*b,const int c[]);""")
     lib = verify(ffi, 'test_const_function_type_args', """
         int (*foobar)(const int a, const int *b, const int c[]);
     """)
@@ -1626,7 +1626,7 @@
 
 def test_extern_python_bogus_name():
     ffi = FFI()
-    ffi.cdef("int abc;")
+    ffi.cdef("extern int abc;")
     lib = verify(ffi, 'test_extern_python_bogus_name', "int abc;")
     def fn():
         pass
@@ -1787,8 +1787,8 @@
     ffi.cdef("""
         extern "Python" int __stdcall foo(int);
         extern "Python" int WINAPI bar(int);
-        int (__stdcall * mycb1)(int);
-        int indirect_call(int);
+        static int (__stdcall * mycb1)(int);
+        static int indirect_call(int);
     """)
     lib = verify(ffi, 'test_extern_python_stdcall', """
         #ifndef _MSC_VER
@@ -1856,7 +1856,7 @@
 
 def test_introspect_global_var():
     ffi = FFI()
-    ffi.cdef("float g1;")
+    ffi.cdef("extern float g1;")
     lib = verify(ffi, 'test_introspect_global_var', """
         float g1;
     """)
@@ -1867,7 +1867,7 @@
 
 def test_introspect_global_var_array():
     ffi = FFI()
-    ffi.cdef("float g1[100];")
+    ffi.cdef("extern float g1[100];")
     lib = verify(ffi, 'test_introspect_global_var_array', """
         float g1[100];
     """)
@@ -2090,7 +2090,7 @@
     ffi = FFI()
     ffi.cdef("""
         typedef int foo_t[...], bar_t[...];
-        int gv[...];
+        extern int gv[...];
         typedef int mat_t[...][...];
         typedef int vmat_t[][...];
         """)
diff --git a/extra_tests/cffi_tests/cffi1/test_verify1.py b/extra_tests/cffi_tests/cffi1/test_verify1.py
--- a/extra_tests/cffi_tests/cffi1/test_verify1.py
+++ b/extra_tests/cffi_tests/cffi1/test_verify1.py
@@ -268,7 +268,7 @@
 def test_var_signed_integer_types():
     ffi = FFI()
     lst = all_signed_integer_types(ffi)
-    csource = "\n".join(["%s somevar_%s;" % (tp, tp.replace(' ', '_'))
+    csource = "\n".join(["static %s somevar_%s;" % (tp, tp.replace(' ', '_'))
                          for tp in lst])
     ffi.cdef(csource)
     lib = ffi.verify(csource)
@@ -287,7 +287,7 @@
 def test_var_unsigned_integer_types():
     ffi = FFI()
     lst = all_unsigned_integer_types(ffi)
-    csource = "\n".join(["%s somevar_%s;" % (tp, tp.replace(' ', '_'))
+    csource = "\n".join(["static %s somevar_%s;" % (tp, tp.replace(' ', '_'))
                          for tp in lst])
     ffi.cdef(csource)
     lib = ffi.verify(csource)
@@ -791,8 +791,8 @@
 
 def test_access_variable():
     ffi = FFI()
-    ffi.cdef("int foo(void);\n"
-             "int somenumber;")
+    ffi.cdef("static int foo(void);\n"
+             "static int somenumber;")
     lib = ffi.verify("""
         static int somenumber = 2;
         static int foo(void) {
@@ -809,7 +809,7 @@
 def test_access_address_of_variable():
     # access the address of 'somenumber': need a trick
     ffi = FFI()
-    ffi.cdef("int somenumber; static int *const somenumberptr;")
+    ffi.cdef("static int somenumber; static int *const somenumberptr;")
     lib = ffi.verify("""
         static int somenumber = 2;
         #define somenumberptr (&somenumber)
@@ -821,8 +821,8 @@
 
 def test_access_array_variable(length=5):
     ffi = FFI()
-    ffi.cdef("int foo(int);\n"
-             "int somenumber[%s];" % (length,))
+    ffi.cdef("static int foo(int);\n"
+             "static int somenumber[%s];" % (length,))
     lib = ffi.verify("""
         static int somenumber[] = {2, 2, 3, 4, 5};
         static int foo(int i) {
@@ -853,8 +853,8 @@
 def test_access_struct_variable():
     ffi = FFI()
     ffi.cdef("struct foo { int x; ...; };\n"
-             "int foo(int);\n"
-             "struct foo stuff;")
+             "static int foo(int);\n"
+             "static struct foo stuff;")
     lib = ffi.verify("""
         struct foo { int x, y, z; };
         static struct foo stuff = {2, 5, 8};
@@ -878,9 +878,9 @@
 
 def test_access_callback():
     ffi = FFI()
-    ffi.cdef("int (*cb)(int);\n"
-             "int foo(int);\n"
-             "void reset_cb(void);")
+    ffi.cdef("static int (*cb)(int);\n"
+             "static int foo(int);\n"
+             "static void reset_cb(void);")
     lib = ffi.verify("""
         static int g(int x) { return x * 7; }
         static int (*cb)(int);
@@ -896,9 +896,9 @@
 def test_access_callback_function_typedef():
     ffi = FFI()
     ffi.cdef("typedef int mycallback_t(int);\n"
-             "mycallback_t *cb;\n"
-             "int foo(int);\n"
-             "void reset_cb(void);")
+             "static mycallback_t *cb;\n"
+             "static int foo(int);\n"
+             "static void reset_cb(void);")
     lib = ffi.verify("""
         static int g(int x) { return x * 7; }
         static int (*cb)(int);
@@ -1039,7 +1039,7 @@
 def test_autofilled_struct_as_argument_dynamic():
     ffi = FFI()
     ffi.cdef("struct foo_s { long a; ...; };\n"
-             "int (*foo)(struct foo_s);")
+             "static int (*foo)(struct foo_s);")
     lib = ffi.verify("""
         struct foo_s {
             double b;
@@ -1048,7 +1048,7 @@
         int foo1(struct foo_s s) {
             return (int)s.a - (int)s.b;
         }
-        int (*foo)(struct foo_s s) = &foo1;
+        static int (*foo)(struct foo_s s) = &foo1;
     """)
     e = py.test.raises(NotImplementedError, lib.foo, "?")
     msg = ("ctype 'struct foo_s' not supported as argument.  It is a struct "
@@ -1424,7 +1424,7 @@
         py.test.skip("_Bool not in MSVC")
     ffi = FFI()
     ffi.cdef("struct foo_s { _Bool x; };"
-             "_Bool foo(_Bool); _Bool (*foop)(_Bool);")
+             "_Bool foo(_Bool); static _Bool (*foop)(_Bool);")
     lib = ffi.verify("""
         struct foo_s { _Bool x; };
         int foo(int arg) {
@@ -1433,7 +1433,7 @@
         _Bool _foofunc(_Bool x) {
             return !x;
         }
-        _Bool (*foop)(_Bool) = _foofunc;
+        static _Bool (*foop)(_Bool) = _foofunc;
     """)
     p = ffi.new("struct foo_s *")
     p.x = 1
@@ -1618,7 +1618,7 @@
 
 def test_FILE_stored_explicitly():
     ffi = FFI()
-    ffi.cdef("int myprintf11(const char *, int); FILE *myfile;")
+    ffi.cdef("int myprintf11(const char *, int); extern FILE *myfile;")
     lib = ffi.verify("""
         #include <stdio.h>
         FILE *myfile;
@@ -1644,13 +1644,13 @@
 
 def test_global_array_with_missing_length():
     ffi = FFI()
-    ffi.cdef("int fooarray[];")
+    ffi.cdef("extern int fooarray[];")
     lib = ffi.verify("int fooarray[50];")
     assert repr(lib.fooarray).startswith("<cdata 'int *'")
 
 def test_global_array_with_dotdotdot_length():
     ffi = FFI()
-    ffi.cdef("int fooarray[...];")
+    ffi.cdef("extern int fooarray[...];")
     lib = ffi.verify("int fooarray[50];")
     assert repr(lib.fooarray).startswith("<cdata 'int[50]'")
 
@@ -1658,7 +1658,7 @@
     py.test.xfail("was detected only because 23 bytes cannot be divided by 4; "
                   "redo more generally")
     ffi = FFI()
-    ffi.cdef("int fooarray[...];")
+    ffi.cdef("extern int fooarray[...];")
     py.test.raises(VerificationError, ffi.verify, "char fooarray[23];")
 
 def test_struct_containing_struct():
@@ -1779,7 +1779,7 @@
 def test_callback_indirection():
     ffi = FFI()
     ffi.cdef("""
-        int (*python_callback)(int how_many, int *values);
+        static int (*python_callback)(int how_many, int *values);
         int (*const c_callback)(int,...);   /* pass this ptr to C routines */
         int some_c_function(int(*cb)(int,...));
     """)
@@ -1913,23 +1913,23 @@
 
 def test_bug_const_char_ptr_array_1():
     ffi = FFI()
-    ffi.cdef("""const char *a[...];""")
+    ffi.cdef("""extern const char *a[...];""")
     lib = ffi.verify("""const char *a[5];""")
     assert repr(ffi.typeof(lib.a)) == "<ctype 'char *[5]'>"
 
 def test_bug_const_char_ptr_array_2():
     ffi = FFI()
-    ffi.cdef("""const int a[];""")
+    ffi.cdef("""extern const int a[];""")
     lib = ffi.verify("""const int a[5];""")
     assert repr(ffi.typeof(lib.a)) == "<ctype 'int *'>"
 
 def _test_various_calls(force_libffi):
     cdef_source = """
-    int xvalue;
-    long long ivalue, rvalue;
-    float fvalue;
-    double dvalue;
-    long double Dvalue;
+    extern int xvalue;
+    extern long long ivalue, rvalue;
+    extern float fvalue;
+    extern double dvalue;
+    extern long double Dvalue;
     signed char tf_bb(signed char x, signed char c);
     unsigned char tf_bB(signed char x, unsigned char c);
     short tf_bh(signed char x, short c);
@@ -2112,7 +2112,7 @@
     old = sys.getdlopenflags()
     try:
         ffi1 = FFI()
-        ffi1.cdef("int foo_verify_dlopen_flags_1;")
+        ffi1.cdef("extern int foo_verify_dlopen_flags_1;")
         sys.setdlopenflags(ffi1.RTLD_GLOBAL | ffi1.RTLD_NOW)
         lib1 = ffi1.verify("int foo_verify_dlopen_flags_1;")
     finally:
@@ -2253,7 +2253,7 @@
 
 def test_macro_var():
     ffi = FFI()
-    ffi.cdef("int myarray[50], my_value;")
+    ffi.cdef("extern int myarray[50], my_value;")
     lib = ffi.verify("""
         int myarray[50];
         int *get_my_value(void) {
diff --git a/extra_tests/cffi_tests/embedding/add1.py b/extra_tests/cffi_tests/embedding/add1.py
--- a/extra_tests/cffi_tests/embedding/add1.py
+++ b/extra_tests/cffi_tests/embedding/add1.py
@@ -12,7 +12,11 @@
     sys.stdout.write("preparing")
     for i in range(3):
         sys.stdout.flush()
-        time.sleep(0.2)
+        # Windows: sometimes time.sleep() doesn't sleep at all.
+        # This appears to occur on recent versions of python only.
+        t_end = time.time() + 0.19
+        while time.time() < t_end:
+            time.sleep(0.2)
         sys.stdout.write(".")
     sys.stdout.write("\n")
 
diff --git a/extra_tests/cffi_tests/embedding/add_recursive.py b/extra_tests/cffi_tests/embedding/add_recursive.py
--- a/extra_tests/cffi_tests/embedding/add_recursive.py
+++ b/extra_tests/cffi_tests/embedding/add_recursive.py
@@ -4,7 +4,7 @@
 ffi = cffi.FFI()
 
 ffi.embedding_api("""
-    int (*my_callback)(int);
+    extern int (*my_callback)(int);
     int add_rec(int, int);
 """)
 
diff --git a/extra_tests/cffi_tests/embedding/test_thread.py b/extra_tests/cffi_tests/embedding/test_thread.py
--- a/extra_tests/cffi_tests/embedding/test_thread.py
+++ b/extra_tests/cffi_tests/embedding/test_thread.py
@@ -22,17 +22,21 @@
         add1_cffi = self.prepare_module('add1')
         add2_cffi = self.prepare_module('add2')
         self.compile('thread2-test', [add1_cffi, add2_cffi], threads=True)
-        output = self.execute('thread2-test')
-        output = self._take_out(output, "preparing")
-        output = self._take_out(output, ".")
-        output = self._take_out(output, ".")
-        # at least the 3rd dot should be after everything from ADD2
-        assert output == ("starting\n"
-                          "prepADD2\n"
-                          "adding 1000 and 200 and 30\n"
-                          ".\n"
-                          "adding 40 and 2\n"
-                          "done\n")
+        for i in range(3):
+            output = self.execute('thread2-test')
+            print('='*79)
+            print(output)
+            print('='*79)
+            output = self._take_out(output, "preparing")
+            output = self._take_out(output, ".")
+            output = self._take_out(output, ".")
+            # at least the 3rd dot should be after everything from ADD2
+            assert output == ("starting\n"
+                              "prepADD2\n"
+                              "adding 1000 and 200 and 30\n"
+                              ".\n"
+                              "adding 40 and 2\n"
+                              "done\n")
 
     def test_alt_issue(self):
         add1_cffi = self.prepare_module('add1')
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -86,7 +86,7 @@
             arch = platform.machine()
         g['LDSHARED'] += ' -undefined dynamic_lookup'
         g['CC'] += ' -arch %s' % (arch,)
-        g['MACOSX_DEPLOYMENT_TARGET'] = '10.14'
+        g['MACOSX_DEPLOYMENT_TARGET'] = '10.7'
 
     global _config_vars
     _config_vars = g
diff --git a/lib-python/2.7/ensurepip/__init__.py b/lib-python/2.7/ensurepip/__init__.py
--- a/lib-python/2.7/ensurepip/__init__.py
+++ b/lib-python/2.7/ensurepip/__init__.py
@@ -12,9 +12,9 @@
 __all__ = ["version", "bootstrap"]
 
 
-_SETUPTOOLS_VERSION = "28.8.0"
+_SETUPTOOLS_VERSION = "41.2.0"
 
-_PIP_VERSION = "9.0.1"
+_PIP_VERSION = "19.2.3"
 
 _PROJECTS = [
     ("setuptools", _SETUPTOOLS_VERSION),
@@ -28,8 +28,8 @@
         sys.path = additional_paths + sys.path
 
     # Install the bundled software
-    import pip
-    pip.main(args)
+    import pip._internal
+    return pip._internal.main(args)
 
 
 def version():
diff --git a/lib-python/2.7/ensurepip/_bundled/pip-19.2.3-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-19.2.3-py2.py3-none-any.whl
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..8118df8ac1940f8c6cb410fbc18e5fae59872b95
GIT binary patch

[cut]
diff --git a/lib-python/2.7/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl
deleted file mode 100644
index 4b8ecc69db7e37fc6dd7b6dd8f690508f42866a1..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch

[cut]

diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl
deleted file mode 100644
index 502e3cb418c154872ad6e677ef8b63557b38ec35..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch

[cut]

diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-41.2.0-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-41.2.0-py2.py3-none-any.whl
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..82df6f63f4ee97380af0a29d8825ae775333b86d
GIT binary patch

[cut]

diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -2,8 +2,18 @@
 import time
 import thread as _thread
 import weakref
-from _pypy_openssl import ffi
-from _pypy_openssl import lib
+
+try:
+    from _pypy_openssl import ffi
+    from _pypy_openssl import lib
+except ImportError as e:
+    import os
+    msg = "\n\nThe _ssl cffi module either doesn't exist or is incompatible with your machine's shared libraries.\n" + \
+          "If you have a compiler installed, you can try to rebuild it by running:\n" + \
+          "cd %s\n" % os.path.abspath(os.path.dirname(os.path.dirname(__file__))) + \
+          "%s _ssl_build.py\n" % sys.executable
+    raise ImportError(str(e) + msg)
+
 from _cffi_ssl._stdssl.certificate import (_test_decode_cert,
     _decode_certificate, _certificate_to_der)
 from _cffi_ssl._stdssl.utility import (_str_with_len, _bytes_with_len,
diff --git a/lib_pypy/_curses_build.py b/lib_pypy/_curses_build.py
--- a/lib_pypy/_curses_build.py
+++ b/lib_pypy/_curses_build.py
@@ -1,14 +1,28 @@
 from cffi import FFI
 import os
 
-# On some systems, the ncurses library is
-# located at /usr/include/ncurses, so we must check this case.
-# Let's iterate over well known paths
-incdirs =  []
-for _path in ['/usr/include', '/usr/include/ncurses']:
-    if os.path.isfile(os.path.join(_path, 'panel.h')):
-        incdirs.append(_path)
-        break
+def find_library(options):
+    for library in options:
+        ffi = FFI()
+        ffi.set_source("_curses_cffi_check", "", libraries=[library])
+        try:
+            ffi.compile()
+        except VerificationError as e:
+            e_last = e
+            continue
+        else:
+            return library
+
+    # If none of the options is available, present the user a meaningful
+    # error message
+    raise e_last
+
+def find_curses_include_dirs():
+    if os.path.exists('/usr/include/ncurses'):
+        return ['/usr/include/ncurses']
+    if os.path.exists('/usr/include/ncursesw'):
+    	return ['/usr/include/ncursesw']
+    return []
 
 
 ffi = FFI()
@@ -59,8 +73,9 @@
 void _m_getsyx(int *yx) {
     getsyx(yx[0], yx[1]);
 }
-""", include_dirs=incdirs, 
-     libraries=['ncurses', 'panel'])
+""", libraries=[find_library(['ncurses', 'ncursesw']),
+                find_library(['panel', 'panelw'])],
+     include_dirs=find_curses_include_dirs())
 
 
 ffi.cdef("""
@@ -71,6 +86,8 @@
 typedef unsigned long... chtype;
 typedef chtype attr_t;
 
+typedef int... wint_t;
+
 typedef struct
 {
     short id;           /* ID to distinguish multiple devices */
@@ -82,6 +99,7 @@
 static const int ERR, OK;
 static const int TRUE, FALSE;
 static const int KEY_MIN, KEY_MAX;
+static const int KEY_CODE_YES;
 
 static const int COLOR_BLACK;
 static const int COLOR_RED;
@@ -168,6 +186,8 @@
 void filter(void);
 int flash(void);
 int flushinp(void);
+int wget_wch(WINDOW *, wint_t *);
+int mvwget_wch(WINDOW *, int, int, wint_t *);
 chtype getbkgd(WINDOW *);
 WINDOW * getwin(FILE *);
 int halfdelay(int);
@@ -243,6 +263,7 @@
 int touchwin(WINDOW *);
 int typeahead(int);
 int ungetch(int);
+int unget_wch(const wchar_t);
 int untouchwin(WINDOW *);
 void use_env(bool);
 int waddch(WINDOW *, const chtype);
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.13.0
+Version: 1.13.1
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -5,8 +5,8 @@
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 from .error import PkgConfigError
 
-__version__ = "1.13.0"
-__version_info__ = (1, 13, 0)
+__version__ = "1.13.1"
+__version_info__ = (1, 13, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -224,7 +224,7 @@
 
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.13.0"
+                               "\ncompiled with cffi version: 1.13.1"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
@@ -327,13 +327,15 @@
 #endif
 
     /* call Py_InitializeEx() */
-    {
-        PyGILState_STATE state = PyGILState_UNLOCKED;
-        if (!Py_IsInitialized())
-            _cffi_py_initialize();
-        else
-            state = PyGILState_Ensure();
-
+    if (!Py_IsInitialized()) {
+        _cffi_py_initialize();
+        PyEval_InitThreads();
+        PyEval_SaveThread();  /* release the GIL */
+        /* the returned tstate must be the one that has been stored into the
+           autoTLSkey by _PyGILState_Init() called from Py_Initialize(). */
+    }
+    else {
+        PyGILState_STATE state = PyGILState_Ensure();
         PyEval_InitThreads();
         PyGILState_Release(state);
     }
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -156,6 +156,13 @@
                           "confuse pre-parsing.")
             break
 
+def _warn_for_non_extern_non_static_global_variable(decl):
+    if not decl.storage:
+        import warnings
+        warnings.warn("Declaration of global variable '%s' in cdef() should "
+                      "be marked 'extern' for consistency (or possibly "
+                      "'static' in API mode)" % (decl.name,))
+
 def _preprocess(csource):
     # Remove comments.  NOTE: this only work because the cdef() section
     # should not contain any string literal!
@@ -506,6 +513,7 @@
                     if (quals & model.Q_CONST) and not tp.is_array_type:
                         self._declare('constant ' + decl.name, tp, quals=quals)
                     else:
+                        _warn_for_non_extern_non_static_global_variable(decl)
                         self._declare('variable ' + decl.name, tp, quals=quals)
 
     def parse_type(self, cdecl):
diff --git a/lib_pypy/pyrepl/historical_reader.py b/lib_pypy/pyrepl/historical_reader.py
--- a/lib_pypy/pyrepl/historical_reader.py
+++ b/lib_pypy/pyrepl/historical_reader.py
@@ -111,7 +111,7 @@
         r.isearch_term = ''
         r.dirty = 1
         r.push_input_trans(r.isearch_trans)
-        
+
 
 class reverse_history_isearch(commands.Command):
     def do(self):
@@ -217,7 +217,7 @@
         self.isearch_trans = input.KeymapTranslator(
             isearch_keymap, invalid_cls=isearch_end,
             character_cls=isearch_add_character)
-        
+
     def select_item(self, i):
         self.transient_history[self.historyi] = self.get_unicode()
         buf = self.transient_history.get(i)
@@ -302,7 +302,7 @@
     reader.ps1 = "h**> "
     reader.ps2 = "h/*> "
     reader.ps3 = "h|*> "
-    reader.ps4 = "h\*> "
+    reader.ps4 = r"h\*> "
     while reader.readline():
         pass
 
diff --git a/lib_pypy/pyrepl/keymap.py b/lib_pypy/pyrepl/keymap.py
--- a/lib_pypy/pyrepl/keymap.py
+++ b/lib_pypy/pyrepl/keymap.py
@@ -58,7 +58,7 @@
     "'":"'",
     '"':'"',
     'a':'\a',
-    'b':'\h',
+    'b':'\b',
     'e':'\033',
     'f':'\f',
     'n':'\n',
diff --git a/lib_pypy/pyrepl/reader.py b/lib_pypy/pyrepl/reader.py
--- a/lib_pypy/pyrepl/reader.py
+++ b/lib_pypy/pyrepl/reader.py
@@ -232,7 +232,7 @@
         self.ps1 = "->> "
         self.ps2 = "/>> "
         self.ps3 = "|.. "
-        self.ps4 = "\__ "
+        self.ps4 = r"\__ "
         self.kill_ring = []
         self.arg = None
         self.finished = 0
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -37,7 +37,7 @@
     "binascii", "_multiprocessing", '_warnings', "_collections",
     "_multibytecodec", "micronumpy", "_continuation", "_cffi_backend",
     "_csv", "_cppyy", "_pypyjson", "_jitlog",
-    #" _ssl", "_hashlib", "crypt"
+    # "_hashlib", "crypt"
 ])
 
 import rpython.rlib.rvmprof.cintf
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -43,6 +43,7 @@
 
 .. toctree::
     whatsnew-pypy3-head.rst
+    whatsnew-pypy3-7.2.0.rst
     whatsnew-pypy3-7.1.0.rst
 
 CPython 3.5 compatible versions
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -9,3 +9,14 @@
 
 Fix segfault when calling descr-methods with no arguments
 
+.. branch: https-readme
+
+Convert http -> https in README.rst
+
+.. branch: license-update
+
+Update list directories in LICENSE
+
+.. branch: allow-forcing-no-embed
+
+When packaging, allow suppressing embedded dependencies via PYPY_NO_EMBED_DEPENDENCIES
diff --git a/pypy/doc/whatsnew-pypy3-5.10.0.rst b/pypy/doc/whatsnew-pypy3-5.10.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy3-5.10.0.rst
@@ -0,0 +1,7 @@
+========================
+What's new in PyPy3 7.0+
+========================
+
+.. this is the revision after release-pypy3.5-v7.0
+.. startrev: 9d2fa7c63b7c
+
diff --git a/pypy/doc/whatsnew-pypy3-6.0.0.rst b/pypy/doc/whatsnew-pypy3-6.0.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy3-6.0.0.rst
@@ -0,0 +1,7 @@
+========================
+What's new in PyPy3 7.0+
+========================
+
+.. this is the revision after release-pypy3.5-v7.0
+.. startrev: 9d2fa7c63b7c
+
diff --git a/pypy/doc/whatsnew-pypy3-7.1.0.rst b/pypy/doc/whatsnew-pypy3-7.1.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy3-7.1.0.rst
@@ -0,0 +1,11 @@
+========================
+What's new in PyPy3 7.0+
+========================
+
+.. this is the revision after release-pypy3.6-v7.0
+.. startrev: 33fe3b2cf186
+
+.. branch: py3.5
+
+Merge in py.35 and use this branch as the primary pypy3 one
+
diff --git a/pypy/doc/whatsnew-pypy3-7.2.0.rst b/pypy/doc/whatsnew-pypy3-7.2.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy3-7.2.0.rst
@@ -0,0 +1,58 @@
+=========================
+What's new in PyPy3 7.2.0
+=========================
+
+.. this is the revision after release-pypy3.6-v7.1.1
+.. startrev: db5a1e7fbbd0
+
+.. branch: fix-literal-prev_digit-underscore
+
+Fix parsing for converting strings with underscore into ints
+
+.. branch: winmultiprocessing
+
+Improve multiprocessing support on win32
+
+.. branch: setitem2d
+
+Allow 2d indexing in ``memoryview.__setitem__`` (issue bb-3028)
+
+.. branch: py3.6-socket-fix
+.. branch: fix-importerror
+.. branch: dj_s390
+.. branch: bpo-35409
+.. branch: remove_array_with_char_test
+.. branch: fix_test_unicode_outofrange
+.. branch: Ram-Rachum/faulthandleris_enabled-should-return-fal-1563636614875
+.. branch: Anthony-Sottile/fix-leak-of-file-descriptor-with-_iofile-1559687440863
+
+.. branch: py3tests
+
+Add handling of application-level test files and -D flag to test runner
+
+.. branch: vendor/stdlib-3.6
+.. branch: stdlib-3.6.9
+
+Update standard library to version 3.6.9
+
+.. branch: __debug__-optimize
+
+Fix handling of __debug__, sys.flags.optimize, and '-O' command-line flag to 
+match CPython 3.6.
+
+.. branch: more-cpyext
+
+Add ``PyErr_SetFromWindowsErr`` and ``pytime.h``, ``pytime.c``. Fix order of
+fields in ``Py_buffer``.
+
+.. branch: Ryan-Hileman/add-support-for-zipfile-stdlib-1562420744699
+
+Add support for the entire stdlib being inside a zipfile
+
+
+.. branch: json-decoder-maps-py3.6
+
+Much faster and more memory-efficient JSON decoding. The resulting
+dictionaries that come out of the JSON decoder have faster lookups too.
+
+
diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -1,7 +1,7 @@
-========================
-What's new in PyPy3 7.0+
-========================
-
-.. this is the revision after release-pypy3.5-v7.0
-.. startrev: 9d2fa7c63b7c
-
+========================
+What's new in PyPy3 7.2+
+========================
+
+.. this is the revision after release-pypy3.6-v7.2
+.. startrev: 6d2f8470165b
+
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -22,9 +22,13 @@
 
 __all__ = ['ObjSpace', 'OperationError', 'W_Root']
 
+def get_printable_location(tp):
+    return "unpackiterable: %s" % (tp, )
+
 unpackiterable_driver = jit.JitDriver(name='unpackiterable',
                                       greens=['tp'],
-                                      reds=['items', 'w_iterator'])
+                                      reds=['items', 'w_iterator'],
+                                      get_printable_location=get_printable_location)
 
 
 class W_Root(object):
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -310,6 +310,9 @@
             return space.w_None
         return tb
 
+    def got_any_traceback(self):
+        return self._application_traceback is not None
+
     def set_traceback(self, traceback):
         """Set the current traceback."""
         self._application_traceback = traceback
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -832,8 +832,20 @@
         w_metaclass = find_metaclass(self.space, w_bases,
                                      w_methodsdict, self.get_w_globals(),
                                      self.get_builtin())
-        w_newclass = self.space.call_function(w_metaclass, w_name,
-                                              w_bases, w_methodsdict)
+        try:
+            w_newclass = self.space.call_function(w_metaclass, w_name,
+                                                  w_bases, w_methodsdict)
+        except OperationError as e:
+            # give a more comprehensible error message for TypeErrors
+            if e.got_any_traceback():
+                raise
+            if not e.match(self.space, self.space.w_TypeError):
+                raise
+            raise oefmt(self.space.w_TypeError,
+                "metaclass found to be '%N', but calling %R "
+                "with args (%R, %R, dict) raised %R",
+                w_metaclass, w_metaclass, w_name, w_bases,
+                e.get_w_value(self.space))
         self.pushvalue(w_newclass)
 
     def STORE_NAME(self, varindex, next_instr):
diff --git a/pypy/interpreter/test/apptest_function.py b/pypy/interpreter/test/apptest_function.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/test/apptest_function.py
@@ -0,0 +1,615 @@
+from pytest import raises, skip
+
+def test_attributes():
+    globals()['__name__'] = 'mymodulename'
+    def f(): pass
+    assert hasattr(f, 'func_code')
+    assert f.func_defaults == None
+    f.func_defaults = None
+    assert f.func_defaults == None
+    assert f.func_dict == {}
+    assert type(f.func_globals) == dict
+    assert f.func_globals is f.__globals__
+    assert f.func_closure is None
+    assert f.func_doc == None
+    assert f.func_name == 'f'
+    assert f.__module__ == 'mymodulename'
+
+def test_code_is_ok():
+    def f(): pass
+    assert not hasattr(f.func_code, '__dict__')
+
+def test_underunder_attributes():
+    def f(): pass
+    assert f.__name__ == 'f'
+    assert f.__doc__ == None
+    assert f.__name__ == f.func_name
+    assert f.__doc__ == f.func_doc
+    assert f.__dict__ is f.func_dict
+    assert f.__code__ is f.func_code
+    assert f.__defaults__ is f.func_defaults
+    assert hasattr(f, '__class__')
+
+def test_classmethod():
+    def f():
+        pass
+    assert classmethod(f).__func__ is f
+    assert staticmethod(f).__func__ is f
+
+def test_write_doc():
+    def f(): "hello"
+    assert f.__doc__ == 'hello'
+    f.__doc__ = 'good bye'
+    assert f.__doc__ == 'good bye'
+    del f.__doc__
+    assert f.__doc__ == None
+
+def test_write_func_doc():
+    def f(): "hello"
+    assert f.func_doc == 'hello'
+    f.func_doc = 'good bye'
+    assert f.func_doc == 'good bye'
+    del f.func_doc
+    assert f.func_doc == None
+
+def test_write_module():
+    def f(): "hello"
+    f.__module__ = 'ab.c'
+    assert f.__module__ == 'ab.c'
+    del f.__module__
+    assert f.__module__ is None
+
+def test_new():
+    def f(): return 42
+    FuncType = type(f)
+    f2 = FuncType(f.func_code, f.func_globals, 'f2', None, None)
+    assert f2() == 42
+
+    def g(x):
+        def f():
+            return x
+        return f
+    f = g(42)
+    with raises(TypeError):
+        FuncType(f.func_code, f.func_globals, 'f2', None, None)
+
+def test_write_code():
+    def f():
+        return 42
+    def g():
+        return 41
+    assert f() == 42
+    assert g() == 41
+    with raises(TypeError):
+        f.func_code = 1
+    f.func_code = g.func_code
+    assert f() == 41
+    def h():
+        return f() # a closure
+    with raises(ValueError):
+        f.func_code = h.func_code
+
+def test_write_code_builtin_forbidden():
+    def f(*args):
+        return 42
+    with raises(TypeError):
+        dir.func_code = f.func_code
+    with raises(TypeError):
+        list.append.im_func.func_code = f.func_code
+
+def test_set_name():
+    def f(): pass
+    f.__name__ = 'g'
+    assert f.func_name == 'g'
+    with raises(TypeError):
+        f.__name__ = u'g'
+
+
+def test_simple_call():
+    def func(arg1, arg2):
+        return arg1, arg2
+    res = func(23,42)
+    assert res[0] == 23
+    assert res[1] == 42
+
+def test_simple_call_default():
+    def func(arg1, arg2=11, arg3=111):
+        return arg1, arg2, arg3
+    res = func(1)
+    assert res[0] == 1
+    assert res[1] == 11
+    assert res[2] == 111
+    res = func(1, 22)
+    assert res[0] == 1
+    assert res[1] == 22
+    assert res[2] == 111
+    res = func(1, 22, 333)
+    assert res[0] == 1
+    assert res[1] == 22
+    assert res[2] == 333
+
+    with raises(TypeError):
+        func()
+    with raises(TypeError):
+        func(1, 2, 3, 4)
+
+def test_simple_varargs():
+    def func(arg1, *args):
+        return arg1, args
+    res = func(23,42)
+    assert res[0] == 23
+    assert res[1] == (42,)
+
+    res = func(23, *(42,))
+    assert res[0] == 23
+    assert res[1] == (42,)
+
+def test_simple_kwargs():
+    def func(arg1, **kwargs):
+        return arg1, kwargs
+    res = func(23, value=42)
+    assert res[0] == 23
+    assert res[1] == {'value': 42}
+
+    res = func(23, **{'value': 42})
+    assert res[0] == 23
+    assert res[1] == {'value': 42}
+
+def test_kwargs_sets_wrong_positional_raises():
+    def func(arg1):
+        pass
+    with raises(TypeError):
+        func(arg2=23)
+
+def test_kwargs_sets_positional():
+    def func(arg1):
+        return arg1
+    res = func(arg1=42)
+    assert res == 42
+
+def test_kwargs_sets_positional_mixed():
+    def func(arg1, **kw):
+        return arg1, kw
+    res = func(arg1=42, something=23)
+    assert res[0] == 42
+    assert res[1] == {'something': 23}
+
+def test_kwargs_sets_positional_twice():
+    def func(arg1, **kw):
+        return arg1, kw
+    with raises(TypeError):
+        func(42, {'arg1': 23})
+
+def test_kwargs_nondict_mapping():
+    class Mapping:
+        def keys(self):
+            return ('a', 'b')
+        def __getitem__(self, key):
+            return key
+    def func(arg1, **kw):
+        return arg1, kw
+    res = func(23, **Mapping())
+    assert res[0] == 23
+    assert res[1] == {'a': 'a', 'b': 'b'}
+    with raises(TypeError) as excinfo:
+        func(42, **[])
+    assert excinfo.value.message == (
+        'argument after ** must be a mapping, not list')
+
+def test_default_arg():
+    def func(arg1,arg2=42):
+        return arg1, arg2
+    res = func(arg1=23)
+    assert res[0] == 23
+    assert res[1] == 42
+
+def test_defaults_keyword_overrides():
+    def func(arg1=42, arg2=23):
+        return arg1, arg2
+    res = func(arg1=23)
+    assert res[0] == 23
+    assert res[1] == 23
+
+def test_defaults_keyword_override_but_leaves_empty_positional():
+    def func(arg1,arg2=42):
+        return arg1, arg2
+    with raises(TypeError):
+        func(arg2=23)
+
+def test_kwargs_disallows_same_name_twice():
+    def func(arg1, **kw):
+        return arg1, kw
+    with raises(TypeError):
+        func(42, **{'arg1': 23})
+
+def test_kwargs_bound_blind():
+    class A(object):
+        def func(self, **kw):
+            return self, kw
+    func = A().func
+    with raises(TypeError):
+        func(self=23)
+    with raises(TypeError):
+        func(**{'self': 23})
+
+def test_kwargs_confusing_name():
+    def func(self):    # 'self' conflicts with the interp-level
+        return self*7  # argument to call_function()
+    res = func(self=6)
+    assert res == 42
+
+def test_get():
+    def func(self): return self
+    obj = object()
+    meth = func.__get__(obj, object)
+    assert meth() == obj
+
+def test_none_get_interaction():
+    skip("XXX issue #2083")
+    assert type(None).__repr__(None) == 'None'
+
+def test_none_get_interaction_2():
+    f = None.__repr__
+    assert f() == 'None'
+
+def test_no_get_builtin():
+    assert not hasattr(dir, '__get__')
+    class A(object):
+        ord = ord
+    a = A()
+    assert a.ord('a') == 97
+
+def test_builtin_as_special_method_is_not_bound():
+    class A(object):
+        __getattr__ = len
+    a = A()
+    assert a.a == 1
+    assert a.ab == 2
+    assert a.abcdefghij == 10
+
+def test_call_builtin():
+    s = 'hello'
+    with raises(TypeError):
+        len()
+    assert len(s) == 5
+    with raises(TypeError):
+        len(s, s)
+    with raises(TypeError):
+        len(s, s, s)
+    assert len(*[s]) == 5
+    assert len(s, *[]) == 5
+    with raises(TypeError):
+        len(some_unknown_keyword=s)
+    with raises(TypeError):
+        len(s, some_unknown_keyword=s)
+    with raises(TypeError):
+        len(s, s, some_unknown_keyword=s)
+
+def test_call_error_message():
+    try:
+        len()
+    except TypeError as e:
+        assert "len() takes exactly 1 argument (0 given)" in e.message
+    else:
+        assert 0, "did not raise"
+
+    try:
+        len(1, 2)
+    except TypeError as e:
+        assert "len() takes exactly 1 argument (2 given)" in e.message
+    else:
+        assert 0, "did not raise"
+
+def test_unicode_docstring():
+    def f():
+        u"hi"
+    assert f.__doc__ == u"hi"
+    assert type(f.__doc__) is unicode
+
+def test_issue1293():
+    def f1(): "doc f1"
+    def f2(): "doc f2"
+    f1.func_code = f2.func_code
+    assert f1.__doc__ == "doc f1"
+
+def test_subclassing():
+    # cannot subclass 'function' or 'builtin_function'
+    def f():
+        pass
+    with raises(TypeError):
+        type('Foo', (type(f),), {})
+    with raises(TypeError):
+        type('Foo', (type(len),), {})
+
+def test_lambda_docstring():
+    # Like CPython, (lambda:"foo") has a docstring of "foo".
+    # But let's not test that.  Just test that (lambda:42) does not
+    # have 42 as docstring.
+    f = lambda: 42
+    assert f.func_doc is None
+
+def test_setstate_called_with_wrong_args():
+    f = lambda: 42
+    # not sure what it should raise, since CPython doesn't have setstate
+    # on function types
+    with raises(ValueError):
+        type(f).__setstate__(f, (1, 2, 3))
+
+def test_simple_call():
+    class A(object):
+        def func(self, arg2):
+            return self, arg2
+    a = A()
+    res = a.func(42)
+    assert res[0] is a
+    assert res[1] == 42
+
+def test_simple_varargs():
+    class A(object):
+        def func(self, *args):
+            return self, args
+    a = A()
+    res = a.func(42)
+    assert res[0] is a
+    assert res[1] == (42,)
+
+    res = a.func(*(42,))
+    assert res[0] is a
+    assert res[1] == (42,)
+
+def test_obscure_varargs():
+    class A(object):
+        def func(*args):
+            return args
+    a = A()
+    res = a.func(42)
+    assert res[0] is a
+    assert res[1] == 42
+
+    res = a.func(*(42,))
+    assert res[0] is a
+    assert res[1] == 42
+
+def test_simple_kwargs():
+    class A(object):
+        def func(self, **kwargs):
+            return self, kwargs
+    a = A()
+
+    res = a.func(value=42)
+    assert res[0] is a
+    assert res[1] == {'value': 42}
+
+    res = a.func(**{'value': 42})
+    assert res[0] is a
+    assert res[1] == {'value': 42}
+
+def test_get():
+    def func(self): return self
+    class Object(object): pass
+    obj = Object()
+    # Create bound method from function
+    obj.meth = func.__get__(obj, Object)
+    assert obj.meth() == obj
+    # Create bound method from method
+    meth2 = obj.meth.__get__(obj, Object)
+    assert meth2() == obj
+
+def test_get_get():
+    # sanxiyn's test from email
+    def m(self): return self
+    class C(object): pass
+    class D(C): pass
+    C.m = m
+    D.m = C.m
+    c = C()
+    assert c.m() == c
+    d = D()
+    assert d.m() == d
+
+def test_method_eq():
+    class C(object):
+        def m(): pass
+    c = C()
+    assert C.m == C.m
+    assert c.m == c.m
+    assert not (C.m == c.m)
+    assert not (c.m == C.m)
+    c2 = C()
+    assert (c.m == c2.m) is False
+    assert (c.m != c2.m) is True
+    assert (c.m != c.m) is False
+
+def test_method_hash():
+    class C(object):
+        def m(): pass
+    class D(C):
+        pass
+    c = C()
+    assert hash(C.m) == hash(D.m)
+    assert hash(c.m) == hash(c.m)
+
+def test_method_repr():
+    class A(object):
+        def f(self):
+            pass
+    assert repr(A.f) == "<unbound method A.f>"
+    assert repr(A().f).startswith("<bound method A.f of <")
+    assert repr(A().f).endswith(">>")
+    class B:
+        def f(self):
+            pass
+    assert repr(B.f) == "<unbound method B.f>"
+    assert repr(B().f).startswith("<bound method B.f of <")
+    assert repr(A().f).endswith(">>")
+
+    assert repr(type(A.f)) == repr(type(A().f)) == "<type 'instancemethod'>"
+
+
+def test_method_call():
+    class C(object):
+        def __init__(self, **kw):
+            pass
+    c = C(type='test')
+
+def test_method_w_callable():
+    class A(object):
+        def __call__(self, x):
+            return x
+    import new
+    im = new.instancemethod(A(), 3)
+    assert im() == 3
+
+def test_method_w_callable_call_function():
+    class A(object):
+        def __call__(self, x, y):
+            return x+y
+    import new
+    im = new.instancemethod(A(), 3)
+    assert map(im, [4]) == [7]
+
+def test_unbound_typecheck():
+    class A(object):
+        def foo(self, *args):
+            return args
+    class B(A):
+        pass
+    class C(A):
+        pass
+
+    assert A.foo(A(), 42) == (42,)
+    assert A.foo(B(), 42) == (42,)
+    with raises(TypeError):
+        A.foo(5)
+    with raises(TypeError):
+        B.foo(C())
+    with raises(TypeError):
+        class Fun:

From pypy.commits at gmail.com  Fri Nov  1 17:14:52 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 01 Nov 2019 14:14:52 -0700 (PDT)
Subject: [pypy-commit] pypy expose-jsonmap: handle repeated keys
Message-ID: <5dbca04c.1c69fb81.e8e9b.b4e1@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: expose-jsonmap
Changeset: r97928:31fef096292c
Date: 2019-11-01 22:14 +0100
http://bitbucket.org/pypy/pypy/changeset/31fef096292c/

Log:	handle repeated keys

diff --git a/pypy/module/__pypy__/interp_dictstructure.py b/pypy/module/__pypy__/interp_dictstructure.py
--- a/pypy/module/__pypy__/interp_dictstructure.py
+++ b/pypy/module/__pypy__/interp_dictstructure.py
@@ -17,6 +17,8 @@
             raise oefmt(space.w_TypeError, "expected unicode, got %T", w_x)
         u = space.utf8_w(w_x)
         m = m.get_next(w_x, u, 0, len(u), terminator)
+        if m is None:
+            raise oefmt(space.w_ValueError, "repeated key %R", w_x)
         if not m.is_state_useful():
             m.mark_useful(terminator)  # XXX too aggressive?
     return m
diff --git a/pypy/module/__pypy__/test/test_dictstructure.py b/pypy/module/__pypy__/test/test_dictstructure.py
--- a/pypy/module/__pypy__/test/test_dictstructure.py
+++ b/pypy/module/__pypy__/test/test_dictstructure.py
@@ -24,3 +24,8 @@
         from __pypy__ import newdictstructure, strategy
         m = newdictstructure([u"a"])
         assert repr(m) == "<DictStructure [u'a']>"
+
+    def test_repeated_key(self):
+        from __pypy__ import newdictstructure
+        with raises(ValueError):
+            newdictstructure([u"a", u"a"])

From pypy.commits at gmail.com  Fri Nov  1 17:43:12 2019
From: pypy.commits at gmail.com (mattip)
Date: Fri, 01 Nov 2019 14:43:12 -0700 (PDT)
Subject: [pypy-commit] pypy default: make print in package.py 3-compatible
Message-ID: <5dbca6f0.1c69fb81.f9751.f84f@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r97929:8f3a6314492e
Date: 2019-11-01 05:51 -0400
http://bitbucket.org/pypy/pypy/changeset/8f3a6314492e/

Log:	make print in package.py 3-compatible

diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+from __future__ import print_function 
 """ packages PyPy, provided that it's already built.
 It uses 'pypy/goal/pypy-c' and parts of the rest of the working
 copy.  Usage:
@@ -21,7 +22,9 @@
 import fnmatch
 import subprocess
 import glob
+import platform
 from pypy.tool.release.smartstrip import smartstrip
+from pypy.tool.release.make_portable import make_portable
 
 USE_ZIPFILE_MODULE = sys.platform == 'win32'
 
@@ -89,10 +92,11 @@
         )
 
         for key, module in failures:
-            print >>sys.stderr, """!!!!!!!!!!\nBuilding {0} bindings failed.
+            print("""!!!!!!!!!!\nBuilding {0} bindings failed.
                 You can either install development headers package,
                 add the --without-{0} option to skip packaging this
-                binary CFFI extension, or say --without-cffi.""".format(key)
+                binary CFFI extension, or say --without-cffi.""".format(key),
+                file=sys.stderr)
         if len(failures) > 0:
             return 1, None
 
@@ -129,7 +133,7 @@
         if pypyw.exists():
             tgt = py.path.local(tgt)
             binaries.append((pypyw, tgt.new(purebasename=tgt.purebasename + 'w').basename, None))
-            print "Picking %s" % str(pypyw)
+            print("Picking %s" % str(pypyw))
         # Can't rename a DLL: it is always called 'libpypy-c.dll'
         win_extras = [('libpypy-c.dll', None), ('sqlite3.dll', lib_pypy)]
         if not options.no_tk:
@@ -141,18 +145,19 @@
             if not p.check():
                 p = py.path.local.sysfind(extra)
                 if not p:
-                    print "%s not found, expect trouble if this is a shared build" % (extra,)
+                    print("%s not found, expect trouble if this "
+                          "is a shared build" % (extra,))
                     continue
-            print "Picking %s" % p
+            print("Picking %s" % p)
             binaries.append((p, p.basename, target_dir))
         libsdir = basedir.join('libs')
         if libsdir.exists():
-            print 'Picking %s (and contents)' % libsdir
+            print('Picking %s (and contents)' % libsdir)
             shutil.copytree(str(libsdir), str(pypydir.join('libs')))
         else:
-            print '"libs" dir with import library not found.'
-            print 'You have to create %r' % (str(libsdir),)
-            print 'and copy libpypy-c.lib in there, renamed to python27.lib'
+            print('"libs" dir with import library not found.')
+            print('You have to create %r' % (str(libsdir),))
+            print('and copy libpypy-c.lib in there, renamed to python27.lib')
             # XXX users will complain that they cannot compile capi (cpyext)
             # modules for windows, also embedding pypy (i.e. in cffi)
             # will fail.
@@ -169,14 +174,15 @@
                 tktcldir = p.dirpath().join('..').join('lib')
                 shutil.copytree(str(tktcldir), str(pypydir.join('tcl')))
             except WindowsError:
-                print >>sys.stderr, r"""Packaging Tk runtime failed.
-tk85.dll and tcl85.dll found in %s, expecting to find runtime in %s
-directory next to the dlls, as per build instructions.""" %(p, tktcldir)
+                print("Packaging Tk runtime failed. tk85.dll and tcl85.dll "
+                      "found in %s, expecting to find runtime in %s directory "
+                      "next to the dlls, as per build "
+                      "instructions." %(p, tktcldir), file=sys.stderr)
                 import traceback;traceback.print_exc()
                 raise MissingDependenciesError('Tk runtime')
 
-    print '* Binaries:', [source.relto(str(basedir))
-                          for source, target, target_dir in binaries]
+    print('* Binaries:', [source.relto(str(basedir))
+                          for source, target, target_dir in binaries])
 
     # Careful: to copy lib_pypy, copying just the hg-tracked files
     # would not be enough: there are also ctypes_config_cache/_*_cache.py.
@@ -243,10 +249,11 @@
         else:
             archive = str(builddir.join(name + '.tar.bz2'))
             if sys.platform == 'darwin':
-                print >>sys.stderr, """Warning: tar on current platform does not suport overriding the uid and gid
-for its contents. The tarball will contain your uid and gid. If you are
-building the actual release for the PyPy website, you may want to be
-using another platform..."""
+                print("Warning: tar on current platform does not suport "
+                      "overriding the uid and gid for its contents. The tarball "
+                      "will contain your uid and gid. If you are building the "
+                      "actual release for the PyPy website, you may want to be "
+                      "using another platform...", file=sys.stderr)
                 e = os.system('tar --numeric-owner -cvjf ' + archive + " " + name)
             elif sys.platform.startswith('freebsd'):
                 e = os.system('tar --uname=root --gname=wheel -cvjf ' + archive + " " + name)
@@ -259,10 +266,10 @@
     finally:
         os.chdir(old_dir)
     if options.targetdir:
-        print "Copying %s to %s" % (archive, options.targetdir)
+        print("Copying %s to %s" % (archive, options.targetdir))
         shutil.copy(archive, options.targetdir)
     else:
-        print "Ready in %s" % (builddir,)
+        print("Ready in %s" % (builddir,))
     return retval, builddir # for tests
 
 def package(*args, **kwds):
@@ -310,7 +317,7 @@
                         dest='embed_dependencies',
                         action=NegateAction,
                         default=(sys.platform == 'darwin'),
-                        help='whether to embed dependencies for distribution '
+                        help='whether to embed dependencies in CFFI modules '
                         '(default on OS X)')
     options = parser.parse_args(args)
 

From pypy.commits at gmail.com  Fri Nov  1 17:43:14 2019
From: pypy.commits at gmail.com (mattip)
Date: Fri, 01 Nov 2019 14:43:14 -0700 (PDT)
Subject: [pypy-commit] pypy default: add a --make-portable option to
 package.py
Message-ID: <5dbca6f2.1c69fb81.2281e.58e8@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r97930:b655ef00bd4b
Date: 2019-11-01 05:52 -0400
http://bitbucket.org/pypy/pypy/changeset/b655ef00bd4b/

Log:	add a --make-portable option to package.py

diff --git a/pypy/tool/release/make_portable.py b/pypy/tool/release/make_portable.py
new file mode 100644
--- /dev/null
+++ b/pypy/tool/release/make_portable.py
@@ -0,0 +1,109 @@
+#!/usr/bin/env python
+
+bundle = ['sqlite3', 'ssl', 'crypto', 'ffi', 'expat', 'tcl', 'tk', 'gdbm', 'lzma', 'ncursesw', 'panelw', 'tinfow']
+
+from os import chdir, mkdir, symlink
+from os.path import dirname, relpath, join, exists, basename, realpath
+from shutil import copy2
+import sys
+from glob import glob
+from subprocess import check_output, check_call
+
+
+def get_deps(binary):
+    deps = {}
+    output = check_output(['ldd', binary])
+    for line in output.splitlines():
+        if '=>' not in line:
+            continue
+        line = line.strip()
+        needed, path = line.split(' => ')
+        if path == 'not found':
+            print('Broken dependency in ' + binary)
+        path = path.split(' ')[0]
+        path = realpath(path)
+        if not path:
+            continue
+
+        if needed[3:].split('.', 1)[0] not in bundle:
+            continue
+
+        deps[needed] = path
+        deps.update(get_deps(path))
+
+    return deps
+
+
+def gather_deps(binaries):
+    deps = {}
+    for binary in binaries:
+        deps.update(get_deps(binary))
+
+    return deps
+
+
+def copy_deps(deps):
+    copied = {}
+
+    for needed, path in deps.items():
+        bname = basename(path)
+
+        copy2(path, 'lib/' + bname)
+        copied[path] = 'lib/' + bname
+
+        if not exists('lib/' + needed):
+            symlink(bname, 'lib/' + needed)
+
+    return copied
+
+
+def rpath_binaries(binaries):
+    rpaths = {}
+
+    for binary in binaries:
+        rpath = join('$ORIGIN', relpath('lib', dirname(binary)))
+        check_call(['patchelf', '--set-rpath', rpath, binary])
+
+        rpaths[binary] = rpath
+
+    return rpaths
+
+
+def make_portable():
+    binaries = glob('bin/libpypy*.so')
+    if not binaries:
+        raise ValueError('Could not find bin/libpypy*.so')
+    binaries.extend(glob('lib_pypy/*_cffi.pypy*.so'))
+    binaries.extend(glob('lib_pypy/_pypy_openssl*.so'))
+    binaries.extend(glob('lib_pypy/_tkinter/*_cffi.pypy*.so'))
+
+    deps = gather_deps(binaries)
+
+    copied = copy_deps(deps)
+
+    for path, item in copied.items():
+        print('Copied {0} to {1}'.format(path, item))
+
+    binaries.extend(copied.values())
+
+    rpaths = rpath_binaries(binaries)
+    for binary, rpath in rpaths.items():
+        print('Set RPATH of {0} to {1}'.format(binary, rpath))
+
+    return deps
+
+
+if __name__ == '__main__':
+    try:
+        chdir(sys.argv[1])
+    except:
+        print('Call as %s <path/to/pypy/topdir' % sys.argv[0])
+        exit(-1)
+
+    try:
+        mkdir('lib')
+    except OSError:
+        pass
+
+    make_portable()
+
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -235,7 +235,11 @@
                 else:
                     archive = bindir.join(target)
                 smartstrip(archive, keep_debug=options.keep_debug)
-        #
+
+            # make the package portable by adding rpath=$ORIGIN/..lib,
+            # bundling dependencies
+            if options.make_portable:
+                make_portable()
         if USE_ZIPFILE_MODULE:
             import zipfile
             archive = str(builddir.join(name + '.zip'))
@@ -319,6 +323,12 @@
                         default=(sys.platform == 'darwin'),
                         help='whether to embed dependencies in CFFI modules '
                         '(default on OS X)')
+    parser.add_argument('--make-portable', '--no-make-portable',
+                        dest='make_portable',
+                        action=NegateAction,
+                        default=(platform.linux_distribution() in ('CentOS',)),
+                        help='whether to make the package portable by shipping '
+                            'dependent shared objects and mangling RPATH')
     options = parser.parse_args(args)
 
     if os.environ.has_key("PYPY_PACKAGE_NOKEEPDEBUG"):
@@ -329,6 +339,10 @@
         options.embed_dependencies = True
     elif os.environ.has_key("PYPY_NO_EMBED_DEPENDENCIES"):
         options.embed_dependencies = False
+    if os.environ.has_key("PYPY_MAKE_PORTABLE"):
+        options.embed_dependencies = True
+    elif os.environ.has_key("PYPY_NO_MAKE_PORTABLE"):
+        options.embed_dependencies = False
     if not options.builddir:
         # The import actually creates the udir directory
         from rpython.tool.udir import udir

From pypy.commits at gmail.com  Fri Nov  1 17:43:16 2019
From: pypy.commits at gmail.com (mattip)
Date: Fri, 01 Nov 2019 14:43:16 -0700 (PDT)
Subject: [pypy-commit] pypy default: add version detection to _curses.
 test_curses.py silently skips when importing fails
Message-ID: <5dbca6f4.1c69fb81.46e52.a0ad@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r97931:883f6c7d4239
Date: 2019-11-01 05:57 -0400
http://bitbucket.org/pypy/pypy/changeset/883f6c7d4239/

Log:	add version detection to _curses. test_curses.py silently skips when
	importing fails

diff --git a/lib_pypy/_curses_build.py b/lib_pypy/_curses_build.py
--- a/lib_pypy/_curses_build.py
+++ b/lib_pypy/_curses_build.py
@@ -1,12 +1,18 @@
 from cffi import FFI
 import os
 
+version_str = '''
+    static const int NCURSES_VERSION_MAJOR;
+    static const int NCURSES_VERSION_MINOR;
+'''
+
 def find_library(options):
     for library in options:
         ffi = FFI()
-        ffi.set_source("_curses_cffi_check", "", libraries=[library])
+        ffi.cdef(version_str)
+        ffi.set_source("_curses_cffi_check", version_str, libraries=[library])
         try:
-            ffi.compile()
+            lib = ffi.compile()
         except VerificationError as e:
             e_last = e
             continue
@@ -26,7 +32,6 @@
 
 
 ffi = FFI()
-
 ffi.set_source("_curses_cffi", """
 #ifdef __APPLE__
 /* the following define is necessary for OS X 10.6+; without it, the
@@ -77,6 +82,9 @@
                 find_library(['panel', 'panelw'])],
      include_dirs=find_curses_include_dirs())
 
+import _curses_cffi_check
+lib = _curses_cffi_check.lib
+version = (lib.NCURSES_VERSION_MAJOR, lib.NCURSES_VERSION_MINOR)
 
 ffi.cdef("""
 typedef ... WINDOW;
@@ -186,8 +194,6 @@
 void filter(void);
 int flash(void);
 int flushinp(void);
-int wget_wch(WINDOW *, wint_t *);
-int mvwget_wch(WINDOW *, int, int, wint_t *);
 chtype getbkgd(WINDOW *);
 WINDOW * getwin(FILE *);
 int halfdelay(int);
@@ -263,7 +269,6 @@
 int touchwin(WINDOW *);
 int typeahead(int);
 int ungetch(int);
-int unget_wch(const wchar_t);
 int untouchwin(WINDOW *);
 void use_env(bool);
 int waddch(WINDOW *, const chtype);
@@ -367,6 +372,13 @@
 void _m_getsyx(int *yx);
 """)
 
+if version > (5, 7):
+    ffi.cdef("""
+int wget_wch(WINDOW *, wint_t *);
+int mvwget_wch(WINDOW *, int, int, wint_t *);
+int unget_wch(const wchar_t);
+""")
+
 
 if __name__ == "__main__":
     ffi.compile()

From pypy.commits at gmail.com  Fri Nov  1 17:43:18 2019
From: pypy.commits at gmail.com (mattip)
Date: Fri, 01 Nov 2019 14:43:18 -0700 (PDT)
Subject: [pypy-commit] pypy default: package more libraries to make curses
 work
Message-ID: <5dbca6f6.1c69fb81.bb608.faf2@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r97932:ff495c204b4f
Date: 2019-11-01 08:52 -0400
http://bitbucket.org/pypy/pypy/changeset/ff495c204b4f/

Log:	package more libraries to make curses work

diff --git a/pypy/tool/release/make_portable.py b/pypy/tool/release/make_portable.py
--- a/pypy/tool/release/make_portable.py
+++ b/pypy/tool/release/make_portable.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 
-bundle = ['sqlite3', 'ssl', 'crypto', 'ffi', 'expat', 'tcl', 'tk', 'gdbm', 'lzma', 'ncursesw', 'panelw', 'tinfow']
+bundle = ['sqlite3', 'ssl', 'crypto', 'ffi', 'expat', 'tcl', 'tk', 'gdbm',
+          'lzma', 'tinfo', 'tinfow', 'ncursesw', 'panelw', 'ncurses', 'panel', 'panelw']
 
 from os import chdir, mkdir, symlink
 from os.path import dirname, relpath, join, exists, basename, realpath

From pypy.commits at gmail.com  Fri Nov  1 20:02:46 2019
From: pypy.commits at gmail.com (Yannick_Jadoul)
Date: Fri, 01 Nov 2019 17:02:46 -0700 (PDT)
Subject: [pypy-commit] pypy py3.7-pep564: Adding ns-resolution function to
 time module
Message-ID: <5dbcc7a6.1c69fb81.fe9a7.e84a@mx.google.com>

Author: Yannick Jadoul <yannick.jadoul at belgacom.net>
Branch: py3.7-pep564
Changeset: r97933:f0362a20d1a1
Date: 2019-10-17 11:56 +0200
http://bitbucket.org/pypy/pypy/changeset/f0362a20d1a1/

Log:	Adding ns-resolution function to time module

diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -132,7 +132,7 @@
         def __init__(self):
             self.n_overflow = 0
             self.last_ticks = 0
-            self.divisor = 0.0
+            self.divisor = 0
             self.counter_start = 0
 
         def check_GetTickCount64(self, *args):
@@ -252,7 +252,7 @@
                                             'GetSystemTimeAdjustment',
                                             [LPDWORD, LPDWORD, rwin32.LPBOOL],
                                             rffi.INT)
-    def gettimeofday(space, w_info=None):
+    def _gettimeofday_impl(space, w_info, return_ns):
         with lltype.scoped_alloc(rwin32.FILETIME) as system_time:
             _GetSystemTimeAsFileTime(system_time)
             quad_part = (system_time.c_dwLowDateTime |
@@ -267,8 +267,6 @@
             offset = (r_ulonglong(16384) * r_ulonglong(27) * r_ulonglong(390625)
                      * r_ulonglong(79) * r_ulonglong(853))
             microseconds = quad_part / 10 - offset
-            tv_sec = microseconds / 1000000
-            tv_usec = microseconds % 1000000
             if w_info:
                 with lltype.scoped_alloc(LPDWORD.TO, 1) as time_adjustment, \
                      lltype.scoped_alloc(LPDWORD.TO, 1) as time_increment, \
@@ -278,7 +276,12 @@
 
                     _setinfo(space, w_info, "GetSystemTimeAsFileTime()",
                              time_increment[0] * 1e-7, False, True)
-            return space.newfloat(tv_sec + tv_usec * 1e-6)
+            if return_ns:
+                return space.newint(microseconds * 10**3)
+            else:
+                tv_sec = microseconds / 10**6
+                tv_usec = microseconds % 10**6
+                return space.newfloat(tv_sec + tv_usec * 1e-6)
 else:
     if HAVE_GETTIMEOFDAY:
         if GETTIMEOFDAY_NO_TZ:
@@ -287,7 +290,7 @@
         else:
             c_gettimeofday = external('gettimeofday',
                                       [lltype.Ptr(TIMEVAL), rffi.VOIDP], rffi.INT)
-    def gettimeofday(space, w_info=None):
+    def _gettimeofday_impl(space, w_info, return_ns):
         if HAVE_GETTIMEOFDAY:
             with lltype.scoped_alloc(TIMEVAL) as timeval:
                 if GETTIMEOFDAY_NO_TZ:
@@ -298,22 +301,38 @@
                 if rffi.cast(rffi.LONG, errcode) == 0:
                     if w_info is not None:
                         _setinfo(space, w_info, "gettimeofday()", 1e-6, False, True)
-                    return space.newfloat(
-                        widen(timeval.c_tv_sec) +
-                        widen(timeval.c_tv_usec) * 1e-6)
+                    if return_ns:
+                        return space.newint(
+                            widen(timeval.c_tv_sec) * 10**9 +
+                            widen(timeval.c_tv_usec) * 10**3)
+                    else:
+                        return space.newfloat(
+                            widen(timeval.c_tv_sec) +
+                            widen(timeval.c_tv_usec) * 1e-6)
         if HAVE_FTIME:
             with lltype.scoped_alloc(TIMEB) as t:
                 c_ftime(t)
-                result = (widen(t.c_time) +
-                          widen(t.c_millitm) * 0.001)
                 if w_info is not None:
-                    _setinfo(space, w_info, "ftime()", 1e-3,
-                             False, True)
-            return space.newfloat(result)
+                    _setinfo(space, w_info, "ftime()", 1e-3, False, True)
+                if return_ns:
+                    return space.newint(
+                        widen(t.c_time) * 10**9 +
+                        widen(t.c_millitm) * 10**6)
+                else:
+                    return space.newfloat(
+                        widen(t.c_time) +
+                        widen(t.c_millitm) * 1e-3)
         else:
             if w_info:
                 _setinfo(space, w_info, "time()", 1.0, False, True)
-            return space.newint(c_time(lltype.nullptr(rffi.TIME_TP.TO)))
+            result = c_time(lltype.nullptr(rffi.TIME_TP.TO))
+            if return_ns:
+                return space.newint(widen(result) * 10**9)
+            else:
+                return space.newint(result)
+    
+    def gettimeofday(space, w_info=None):
+        return _gettimeofday_impl(space, w_info, False)
 
 TM_P = lltype.Ptr(tm)
 c_time = external('time', [rffi.TIME_TP], rffi.TIME_T)
@@ -663,11 +682,7 @@
     if not 0 <= rffi.getintfield(t_ref, 'c_tm_yday') <= 365:
         raise oefmt(space.w_ValueError, "day of year out of range")
 
-def time(space, w_info=None):
-    """time() -> floating point number
-
-    Return the current time in seconds since the Epoch.
-    Fractions of a second may be present if the system clock provides them."""
+def _time_impl(space, w_info, return_ns):
     if HAS_CLOCK_GETTIME:
         with lltype.scoped_alloc(TIMESPEC) as timespec:
             ret = c_clock_gettime(rtime.CLOCK_REALTIME, timespec)
@@ -681,8 +696,24 @@
                             res = 1e-9
                         _setinfo(space, w_info, "clock_gettime(CLOCK_REALTIME)",
                                  res, False, True)
-                return space.newfloat(_timespec_to_seconds(timespec))
-    return gettimeofday(space, w_info)
+                if return_ns:
+                    return space.newint(_timespec_to_nanoseconds(timespec))
+                else:
+                    return space.newfloat(_timespec_to_seconds(timespec))
+    return _gettimeofday_impl(space, w_info, return_ns)
+
+def time(space, w_info=None):
+    """time() -> floating point number
+
+    Return the current time in seconds since the Epoch.
+    Fractions of a second may be present if the system clock provides them."""
+    return _time_impl(space, w_info, False)
+
+def time_ns(space, w_info=None):
+    """time_ns() -> int
+    
+    Return the current time in nanoseconds since the Epoch."""
+    return _time_impl(space, w_info, True)
 
 def ctime(space, w_seconds=None):
     """ctime([seconds]) -> string
@@ -789,14 +820,26 @@
     def _timespec_to_seconds(timespec):
         return widen(timespec.c_tv_sec) + widen(timespec.c_tv_nsec) * 1e-9
 
-    @unwrap_spec(clk_id='c_int')
-    def clock_gettime(space, clk_id):
+    def _timespec_to_nanoseconds(timespec):
+        return widen(timespec.c_tv_sec) * 10**9 + widen(timespec.c_tv_nsec)
+    
+    def _clock_gettime_impl(space, clk_id, return_ns):
         with lltype.scoped_alloc(TIMESPEC) as timespec:
             ret = c_clock_gettime(clk_id, timespec)
             if ret != 0:
                 raise exception_from_saved_errno(space, space.w_OSError)
-            secs = _timespec_to_seconds(timespec)
-        return space.newfloat(secs)
+            if return_ns:
+                return space.newint(_timespec_to_nanoseconds(timespec))
+            else:
+                return space.newfloat(_timespec_to_seconds(timespec))
+
+    @unwrap_spec(clk_id='c_int')
+    def clock_gettime(space, clk_id):
+        return _clock_gettime_impl(space, clk_id, False)
+
+    @unwrap_spec(clk_id='c_int')
+    def clock_gettime_ns(space, clk_id):
+        return _clock_gettime_impl(space, clk_id, True)
 
     @unwrap_spec(clk_id='c_int', secs=float)
     def clock_settime(space, clk_id, secs):
@@ -809,6 +852,15 @@
             if ret != 0:
                 raise exception_from_saved_errno(space, space.w_OSError)
 
+    @unwrap_spec(clk_id='c_int', ns=int)
+    def clock_settime_ns(space, clk_id, ns):
+        with lltype.scoped_alloc(TIMESPEC) as timespec:
+            rffi.setintfield(timespec, 'c_tv_sec', ns // 10**9)
+            rffi.setintfield(timespec, 'c_tv_nsec', ns % 10**9)
+            ret = c_clock_settime(clk_id, timespec)
+            if ret != 0:
+                raise exception_from_saved_errno(space, space.w_OSError)
+
     @unwrap_spec(clk_id='c_int')
     def clock_getres(space, clk_id):
         with lltype.scoped_alloc(TIMESPEC) as timespec:
@@ -893,19 +945,18 @@
 if HAS_MONOTONIC:
     if _WIN:
         _GetTickCount = rwin32.winexternal('GetTickCount', [], rwin32.DWORD)
-        def monotonic(space, w_info=None):
+        def _monotonic_impl(space, w_info, return_ns):
             result = 0
             HAS_GETTICKCOUNT64 = time_state.check_GetTickCount64()
             if HAS_GETTICKCOUNT64:
-                result = time_state.GetTickCount64() * 1e-3
+                tick_count = time_state.GetTickCount64()
             else:
                 ticks = _GetTickCount()
                 if ticks < time_state.last_ticks:
                     time_state.n_overflow += 1
                 time_state.last_ticks = ticks
-                result = math.ldexp(time_state.n_overflow, 32)
-                result = result + ticks
-                result = result * 1e-3
+                tick_count = math.ldexp(time_state.n_overflow, 32)
+                tick_count = tick_count + ticks
 
             if w_info is not None:
                 if HAS_GETTICKCOUNT64:
@@ -925,7 +976,11 @@
                             rwin32.lastSavedWindowsError("GetSystemTimeAdjustment"))
                     resolution = resolution * time_increment[0]
                 _setinfo(space, w_info, implementation, resolution, True, False)
-            return space.newfloat(result)
+
+            if return_ns:
+                return space.newint(widen(tick_count) * 10**6)
+            else:
+                return space.newfloat(tick_count * 1e-3)
 
     elif _MACOSX:
         c_mach_timebase_info = external('mach_timebase_info',
@@ -936,7 +991,7 @@
         timebase_info = lltype.malloc(cConfig.TIMEBASE_INFO, flavor='raw',
                                       zero=True, immortal=True)
 
-        def monotonic(space, w_info=None):
+        def _monotonic_impl(space, w_info, return_ns):
             if rffi.getintfield(timebase_info, 'c_denom') == 0:
                 c_mach_timebase_info(timebase_info)
             time = rffi.cast(lltype.Signed, c_mach_absolute_time())
@@ -946,20 +1001,23 @@
             if w_info is not None:
                 res = (numer / denom) * 1e-9
                 _setinfo(space, w_info, "mach_absolute_time()", res, True, False)
-            secs = nanosecs / 10**9
-            rest = nanosecs % 10**9
-            return space.newfloat(float(secs) + float(rest) * 1e-9)
+            if return_ns:
+                return space.newint(nanosecs)
+            else:
+                secs = nanosecs / 10**9
+                rest = nanosecs % 10**9
+                return space.newfloat(float(secs) + float(rest) * 1e-9)
 
     else:
         assert _POSIX
-        def monotonic(space, w_info=None):
+        def _monotonic_impl(space, w_info, return_ns):
             if rtime.CLOCK_HIGHRES is not None:
                 clk_id = rtime.CLOCK_HIGHRES
                 implementation = "clock_gettime(CLOCK_HIGHRES)"
             else:
                 clk_id = rtime.CLOCK_MONOTONIC
                 implementation = "clock_gettime(CLOCK_MONOTONIC)"
-            w_result = clock_gettime(space, clk_id)
+            w_result = _clock_gettime_impl(space, clk_id, return_ns)
             if w_info is not None:
                 with lltype.scoped_alloc(TIMESPEC) as tsres:
                     ret = c_clock_getres(clk_id, tsres)
@@ -970,6 +1028,12 @@
                 _setinfo(space, w_info, implementation, res, True, False)
             return w_result
 
+    def monotonic(space, w_info=None):
+        return _monotonic_impl(space, w_info, False)
+
+    def monotonic_ns(space, w_info=None):
+        return _monotonic_impl(space, w_info, True)
+
 if _WIN:
     # hacking to avoid LARGE_INTEGER which is a union...
     QueryPerformanceCounter = external(
@@ -981,46 +1045,55 @@
     QueryPerformanceFrequency = rwin32.winexternal(
         'QueryPerformanceFrequency', [rffi.CArrayPtr(lltype.SignedLongLong)],
         rffi.INT)
-    def win_perf_counter(space, w_info=None):
+    def _win_perf_counter_impl(space, w_info, return_ns):
         with lltype.scoped_alloc(rffi.CArray(rffi.lltype.SignedLongLong), 1) as a:
             succeeded = True
-            if time_state.divisor == 0.0:
+            if time_state.divisor == 0:
                 QueryPerformanceCounter(a)
                 time_state.counter_start = a[0]
                 succeeded = QueryPerformanceFrequency(a)
-                time_state.divisor = float(a[0])
-            if succeeded and time_state.divisor != 0.0:
+                time_state.divisor = a[0]
+            if succeeded and time_state.divisor != 0:
                 QueryPerformanceCounter(a)
                 diff = a[0] - time_state.counter_start
             else:
                 raise ValueError("Failed to generate the result.")
-            resolution = 1 / time_state.divisor
             if w_info is not None:
+                resolution = 1 / float(time_state.divisor)
                 _setinfo(space, w_info, "QueryPerformanceCounter()", resolution,
                          True, False)
-            return space.newfloat(float(diff) / time_state.divisor)
+            if return_ns:
+                return space.newint(widen(diff) * 10**9 // time_state.divisor)
+            else:
+                return space.newfloat(float(diff) / float(time_state.divisor))
 
-    def perf_counter(space, w_info=None):
+    def _perf_counter_impl(space, w_info, return_ns):
         try:
-            return win_perf_counter(space, w_info=w_info)
+            return _win_perf_counter_impl(space, w_info, return_ns)
         except ValueError:
             if HAS_MONOTONIC:
                 try:
-                    return monotonic(space, w_info=w_info)
+                    return _monotonic_impl(space, w_info, return_ns)
                 except Exception:
                     pass
-        return time(space, w_info=w_info)
+        return _time_impl(space, w_info, return_ns)
 else:
-    def perf_counter(space, w_info=None):
+    def _perf_counter_impl(space, w_info, return_ns):
         if HAS_MONOTONIC:
             try:
-                return monotonic(space, w_info=w_info)
+                return _monotonic_impl(space, w_info, return_ns)
             except Exception:
                 pass
-        return time(space, w_info=w_info)
+        return _time_impl(space, w_info, return_ns)
+
+def perf_counter(space, w_info=None):
+    return _perf_counter_impl(space, w_info, False)
+
+def perf_counter_ns(space, w_info=None):
+    return _perf_counter_impl(space, w_info, True)
 
 if _WIN:
-    def process_time(space, w_info=None):
+    def _process_time_impl(space, w_info, return_ns):
         from rpython.rlib.rposix import GetCurrentProcess, GetProcessTimes
         current_process = GetCurrentProcess()
         with lltype.scoped_alloc(rwin32.FILETIME) as creation_time, \
@@ -1038,11 +1111,14 @@
                           r_ulonglong(user_time.c_dwHighDateTime) << 32)
         if w_info is not None:
             _setinfo(space, w_info, "GetProcessTimes()", 1e-7, True, False)
-        return space.newfloat((float(kernel_time2) + float(user_time2)) * 1e-7)
+        if return_ns:
+            return space.newint((widen(kernel_time2) + widen(user_time2)) * 10**2)
+        else:
+            return space.newfloat((float(kernel_time2) + float(user_time2)) * 1e-7)
 else:
     have_times = hasattr(rposix, 'c_times')
 
-    def process_time(space, w_info=None):
+    def _process_time_impl(space, w_info, return_ns):
         if HAS_CLOCK_GETTIME and (
                 rtime.CLOCK_PROF is not None or
                 rtime.CLOCK_PROCESS_CPUTIME_ID is not None):
@@ -1064,44 +1140,53 @@
                                 res = 1e-9
                         _setinfo(space, w_info,
                                  implementation, res, True, False)
-                    return space.newfloat(_timespec_to_seconds(timespec))
+                    if return_ns:
+                        return space.newint(_timespec_to_nanoseconds(timespec))
+                    else:
+                        return space.newfloat(_timespec_to_seconds(timespec))
 
         if True: # XXX available except if it isn't?
             from rpython.rlib.rtime import (c_getrusage, RUSAGE, RUSAGE_SELF,
-                                            decode_timeval)
+                                            decode_timeval, decode_timeval_ns)
             with lltype.scoped_alloc(RUSAGE) as rusage:
                 ret = c_getrusage(RUSAGE_SELF, rusage)
                 if ret == 0:
                     if w_info is not None:
                         _setinfo(space, w_info,
                                  "getrusage(RUSAGE_SELF)", 1e-6, True, False)
-                    return space.newfloat(decode_timeval(rusage.c_ru_utime) +
-                                          decode_timeval(rusage.c_ru_stime))
+                    if return_ns:
+                        return space.newint(decode_timeval_ns(rusage.c_ru_utime) +
+                                            decode_timeval_ns(rusage.c_ru_stime))
+                    else:
+                        return space.newfloat(decode_timeval(rusage.c_ru_utime) +
+                                              decode_timeval(rusage.c_ru_stime))
         if have_times:
             with lltype.scoped_alloc(rposix.TMS) as tms:
                 ret = rposix.c_times(tms)
                 if rffi.cast(lltype.Signed, ret) != -1:
-                    cpu_time = float(rffi.cast(lltype.Signed,
-                                               tms.c_tms_utime) +
-                                     rffi.cast(lltype.Signed,
-                                               tms.c_tms_stime))
+                    cpu_time = (rffi.cast(lltype.Signed, tms.c_tms_utime) +
+                                rffi.cast(lltype.Signed, tms.c_tms_stime))
                     if w_info is not None:
                         _setinfo(space, w_info, "times()",
                                  1.0 / rposix.CLOCK_TICKS_PER_SECOND,
                                  True, False)
-                    return space.newfloat(cpu_time / rposix.CLOCK_TICKS_PER_SECOND)
-        return clock(space)
+                    if return_ns:
+                        return space.newint(widen(cpu_time) * 10**9 // int(rposix.CLOCK_TICKS_PER_SECOND))
+                    else:
+                        return space.newfloat(float(cpu_time) / rposix.CLOCK_TICKS_PER_SECOND)
+        return _clock_impl(space, w_info, return_ns)
+
+def process_time(space, w_info=None):
+    return _process_time_impl(space, w_info, False)
+
+def process_time_ns(space, w_info=None):
+    return _process_time_impl(space, w_info, True)
 
 _clock = external('clock', [], rposix.CLOCK_T)
-def clock(space, w_info=None):
-    """clock() -> floating point number
-
-    Return the CPU time or real time since the start of the process or since
-    the first call to clock().  This has as much precision as the system
-    records."""
+def _clock_impl(space, w_info, return_ns):
     if _WIN:
         try:
-            return win_perf_counter(space, w_info=w_info)
+            return _win_perf_counter_impl(space, w_info, return_ns)
         except ValueError:
             pass
     value = widen(_clock())
@@ -1112,7 +1197,18 @@
     if w_info is not None:
         _setinfo(space, w_info,
                  "clock()", 1.0 / CLOCKS_PER_SEC, True, False)
-    return space.newfloat(float(value) / CLOCKS_PER_SEC)
+    if return_ns:
+        return space.newint(value * 10**9 // CLOCKS_PER_SEC)
+    else:
+        return space.newfloat(float(value) / CLOCKS_PER_SEC)
+
+def clock(space, w_info=None):
+    """clock() -> floating point number
+
+    Return the CPU time or real time since the start of the process or since
+    the first call to clock().  This has as much precision as the system
+    records."""
+    return _clock_impl(space, w_info, False)
 
 
 def _setinfo(space, w_info, impl, res, mono, adj):
diff --git a/pypy/module/time/moduledef.py b/pypy/module/time/moduledef.py
--- a/pypy/module/time/moduledef.py
+++ b/pypy/module/time/moduledef.py
@@ -11,6 +11,7 @@
 
     interpleveldefs = {
         'time': 'interp_time.time',
+        'time_ns': 'interp_time.time_ns',
         'clock': 'interp_time.clock',
         'ctime': 'interp_time.ctime',
         'asctime': 'interp_time.asctime',
@@ -21,18 +22,23 @@
         'sleep' : 'interp_time.sleep',
         '_STRUCT_TM_ITEMS': 'space.wrap(interp_time._STRUCT_TM_ITEMS)',
         'perf_counter': 'interp_time.perf_counter',
+        'perf_counter_ns': 'interp_time.perf_counter_ns',
         'process_time': 'interp_time.process_time',
+        'process_time_ns': 'interp_time.process_time_ns',
     }
 
     if rtime.HAS_CLOCK_GETTIME:
         interpleveldefs['clock_gettime'] = 'interp_time.clock_gettime'
+        interpleveldefs['clock_gettime_ns'] = 'interp_time.clock_gettime_ns'
         interpleveldefs['clock_settime'] = 'interp_time.clock_settime'
+        interpleveldefs['clock_settime_ns'] = 'interp_time.clock_settime_ns'
         interpleveldefs['clock_getres'] = 'interp_time.clock_getres'
         for constant in rtime.ALL_DEFINED_CLOCKS:
             interpleveldefs[constant] = 'space.wrap(%d)' % (
                 getattr(rtime, constant),)
     if HAS_MONOTONIC:
         interpleveldefs['monotonic'] = 'interp_time.monotonic'
+        interpleveldefs['monotonic_ns'] = 'interp_time.monotonic_ns'
     if os.name == "posix":
         interpleveldefs['tzset'] = 'interp_time.tzset'
 
diff --git a/rpython/rlib/rtime.py b/rpython/rlib/rtime.py
--- a/rpython/rlib/rtime.py
+++ b/rpython/rlib/rtime.py
@@ -9,7 +9,7 @@
 from rpython.rtyper.tool import rffi_platform
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib.objectmodel import register_replacement_for
-from rpython.rlib.rarithmetic import intmask, UINT_MAX
+from rpython.rlib.rarithmetic import intmask, UINT_MAX, widen
 from rpython.rlib import rposix
 
 _WIN32 = sys.platform.startswith('win')
@@ -94,6 +94,10 @@
     return (float(rffi.getintfield(t, 'c_tv_sec')) +
             float(rffi.getintfield(t, 'c_tv_usec')) * 0.000001)
 
+def decode_timeval_ns(t):
+    return (widen(rffi.getintfield(t, 'c_tv_sec')) * 10**9 +
+            widen(rffi.getintfield(t, 'c_tv_usec')) * 10**3)
+
 
 def external(name, args, result, compilation_info=eci, **kwds):
     return rffi.llexternal(name, args, result,

From pypy.commits at gmail.com  Fri Nov  1 20:02:48 2019
From: pypy.commits at gmail.com (Yannick_Jadoul)
Date: Fri, 01 Nov 2019 17:02:48 -0700 (PDT)
Subject: [pypy-commit] pypy py3.7-pep564: Docstrings for time functions
Message-ID: <5dbcc7a8.1c69fb81.f5a85.535a@mx.google.com>

Author: Yannick Jadoul <yannick.jadoul at belgacom.net>
Branch: py3.7-pep564
Changeset: r97934:fb46c09b060b
Date: 2019-10-17 16:51 +0200
http://bitbucket.org/pypy/pypy/changeset/fb46c09b060b/

Log:	Docstrings for time functions

diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -835,14 +835,23 @@
 
     @unwrap_spec(clk_id='c_int')
     def clock_gettime(space, clk_id):
+        """clock_gettime(clk_id) -> float
+    
+        Return the time of the specified clock clk_id."""
         return _clock_gettime_impl(space, clk_id, False)
 
     @unwrap_spec(clk_id='c_int')
     def clock_gettime_ns(space, clk_id):
+        """clock_gettime_ns(clk_id) -> int
+    
+        Return the time of the specified clock clk_id as nanoseconds."""
         return _clock_gettime_impl(space, clk_id, True)
 
     @unwrap_spec(clk_id='c_int', secs=float)
     def clock_settime(space, clk_id, secs):
+        """clock_settime(clk_id, time)
+    
+        Set the time of the specified clock clk_id."""
         with lltype.scoped_alloc(TIMESPEC) as timespec:
             integer_secs = rffi.cast(TIMESPEC.c_tv_sec, secs)
             frac = secs - widen(integer_secs)
@@ -854,6 +863,9 @@
 
     @unwrap_spec(clk_id='c_int', ns=int)
     def clock_settime_ns(space, clk_id, ns):
+        """clock_settime_ns(clk_id, time)
+    
+        Set the time of the specified clock clk_id with nanoseconds."""
         with lltype.scoped_alloc(TIMESPEC) as timespec:
             rffi.setintfield(timespec, 'c_tv_sec', ns // 10**9)
             rffi.setintfield(timespec, 'c_tv_nsec', ns % 10**9)
@@ -863,6 +875,9 @@
 
     @unwrap_spec(clk_id='c_int')
     def clock_getres(space, clk_id):
+        """clock_getres(clk_id) -> floating point number
+    
+        Return the resolution (precision) of the specified clock clk_id."""
         with lltype.scoped_alloc(TIMESPEC) as timespec:
             ret = c_clock_getres(clk_id, timespec)
             if ret != 0:
@@ -1029,9 +1044,15 @@
             return w_result
 
     def monotonic(space, w_info=None):
+        """monotonic() -> float
+    
+        Monotonic clock, cannot go backward."""
         return _monotonic_impl(space, w_info, False)
 
     def monotonic_ns(space, w_info=None):
+        """monotonic_ns() -> int
+    
+        Monotonic clock, cannot go backward, as nanoseconds."""
         return _monotonic_impl(space, w_info, True)
 
 if _WIN:
@@ -1087,9 +1108,15 @@
         return _time_impl(space, w_info, return_ns)
 
 def perf_counter(space, w_info=None):
+    """perf_counter() -> float
+    
+    Performance counter for benchmarking."""
     return _perf_counter_impl(space, w_info, False)
 
 def perf_counter_ns(space, w_info=None):
+    """perf_counter_ns() -> int
+    
+    Performance counter for benchmarking as nanoseconds."""
     return _perf_counter_impl(space, w_info, True)
 
 if _WIN:
@@ -1177,9 +1204,16 @@
         return _clock_impl(space, w_info, return_ns)
 
 def process_time(space, w_info=None):
+    """process_time() -> float
+    
+    Process time for profiling: sum of the kernel and user-space CPU time."""
     return _process_time_impl(space, w_info, False)
 
 def process_time_ns(space, w_info=None):
+    """process_time() -> int
+    
+    Process time for profiling as nanoseconds:
+    sum of the kernel and user-space CPU time"""
     return _process_time_impl(space, w_info, True)
 
 _clock = external('clock', [], rposix.CLOCK_T)

From pypy.commits at gmail.com  Fri Nov  1 20:02:50 2019
From: pypy.commits at gmail.com (Yannick_Jadoul)
Date: Fri, 01 Nov 2019 17:02:50 -0700 (PDT)
Subject: [pypy-commit] pypy py3.7-pep564: Added tests for
 nanosecond-resolution time functions
Message-ID: <5dbcc7aa.1c69fb81.db315.9f7d@mx.google.com>

Author: Yannick Jadoul <yannick.jadoul at belgacom.net>
Branch: py3.7-pep564
Changeset: r97935:7798b261cc60
Date: 2019-10-17 17:52 +0200
http://bitbucket.org/pypy/pypy/changeset/7798b261cc60/

Log:	Added tests for nanosecond-resolution time functions

diff --git a/pypy/module/time/test/test_time.py b/pypy/module/time/test/test_time.py
--- a/pypy/module/time/test/test_time.py
+++ b/pypy/module/time/test/test_time.py
@@ -28,12 +28,22 @@
     def test_time(self):
         import time
         t1 = time.time()
-        assert isinstance(time.time(), float)
-        assert time.time() != 0.0 # 0.0 means failure
+        assert isinstance(t1, float)
+        assert t1 != 0.0  # 0.0 means failure
         time.sleep(0.02)
         t2 = time.time()
         assert t1 != t2       # the resolution should be at least 0.01 secs
 
+    def test_time_ns(self):
+        import time
+        t1 = time.time_ns()
+        assert isinstance(t1, int)
+        assert t1 != 0  # 0 means failure
+        time.sleep(0.02)
+        t2 = time.time_ns()
+        assert t1 != t2       # the resolution should be at least 0.01 secs
+        assert abs(time.time() - time.time_ns() * 1e-9) < 0.1
+
     def test_clock_realtime(self):
         import time
         if not hasattr(time, 'clock_gettime'):
@@ -44,6 +54,18 @@
         t2 = time.clock_gettime(time.CLOCK_REALTIME)
         assert t1 != t2
 
+    def test_clock_realtime_ns(self):
+        import time
+        if not hasattr(time, 'clock_gettime_ns'):
+            skip("need time.clock_gettime_ns()")
+        t1 = time.clock_gettime_ns(time.CLOCK_REALTIME)
+        assert isinstance(t1, int)
+        time.sleep(time.clock_getres(time.CLOCK_REALTIME))
+        t2 = time.clock_gettime_ns(time.CLOCK_REALTIME)
+        assert t1 != t2
+        assert abs(time.clock_gettime(time.CLOCK_REALTIME) -
+                   time.clock_gettime_ns(time.CLOCK_REALTIME) * 1e-9) < 0.1
+
     def test_clock_monotonic(self):
         import time
         if not (hasattr(time, 'clock_gettime') and
@@ -55,6 +77,19 @@
         t2 = time.clock_gettime(time.CLOCK_MONOTONIC)
         assert t1 < t2
 
+    def test_clock_monotonic_ns(self):
+        import time
+        if not (hasattr(time, 'clock_gettime_ns') and
+                hasattr(time, 'CLOCK_MONOTONIC')):
+            skip("need time.clock_gettime()/CLOCK_MONOTONIC")
+        t1 = time.clock_gettime_ns(time.CLOCK_MONOTONIC)
+        assert isinstance(t1, int)
+        time.sleep(time.clock_getres(time.CLOCK_MONOTONIC))
+        t2 = time.clock_gettime_ns(time.CLOCK_MONOTONIC)
+        assert t1 < t2
+        assert abs(time.clock_gettime(time.CLOCK_MONOTONIC) -
+                   time.clock_gettime_ns(time.CLOCK_MONOTONIC) * 1e-9) < 0.1
+
     def test_ctime(self):
         import time
         raises(TypeError, time.ctime, "foo")
@@ -396,10 +431,24 @@
         t2 = time.monotonic()
         assert t1 < t2
 
+    def test_monotonic_ns(self):
+        import time
+        t1 = time.monotonic_ns()
+        assert isinstance(t1, int)
+        time.sleep(0.02)
+        t2 = time.monotonic_ns()
+        assert t1 < t2
+        assert abs(time.monotonic() - time.monotonic_ns() * 1e-9) < 0.1
+
     def test_perf_counter(self):
         import time
         assert isinstance(time.perf_counter(), float)
 
+    def test_perf_counter_ns(self):
+        import time
+        assert isinstance(time.perf_counter_ns(), int)
+        assert abs(time.perf_counter() - time.perf_counter_ns() * 1e-9) < 0.1
+
     def test_process_time(self):
         import time
         t1 = time.process_time()
@@ -409,6 +458,16 @@
         # process_time() should not include time spent during sleep
         assert (t2 - t1) < 0.05
 
+    def test_process_time_ns(self):
+        import time
+        t1 = time.process_time_ns()
+        assert isinstance(t1, int)
+        time.sleep(0.1)
+        t2 = time.process_time_ns()
+        # process_time_ns() should not include time spent during sleep
+        assert (t2 - t1) < 5 * 10**7
+        assert abs(time.process_time() - time.process_time_ns() * 1e-9) < 0.1
+
     def test_get_clock_info(self):
         import time
         clocks = ['clock', 'perf_counter', 'process_time', 'time']

From pypy.commits at gmail.com  Fri Nov  1 20:02:51 2019
From: pypy.commits at gmail.com (Yannick_Jadoul)
Date: Fri, 01 Nov 2019 17:02:51 -0700 (PDT)
Subject: [pypy-commit] pypy py3.7-pep564: Fix return type of edge-case of
 gettimeofday when only C's time() is available
Message-ID: <5dbcc7ab.1c69fb81.fe9a7.e84d@mx.google.com>

Author: Yannick Jadoul <yannick.jadoul at belgacom.net>
Branch: py3.7-pep564
Changeset: r97936:9741d65e2e03
Date: 2019-10-17 18:41 +0200
http://bitbucket.org/pypy/pypy/changeset/9741d65e2e03/

Log:	Fix return type of edge-case of gettimeofday when only C's time() is
	available

diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -329,7 +329,7 @@
             if return_ns:
                 return space.newint(widen(result) * 10**9)
             else:
-                return space.newint(result)
+                return space.newfloat(float(result))
     
     def gettimeofday(space, w_info=None):
         return _gettimeofday_impl(space, w_info, False)

From pypy.commits at gmail.com  Fri Nov  1 21:04:11 2019
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 01 Nov 2019 18:04:11 -0700 (PDT)
Subject: [pypy-commit] pypy py3.7: Add sys._framework attribute
Message-ID: <5dbcd60b.1c69fb81.46e52.bf4d@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.7
Changeset: r97937:48c567ce1caf
Date: 2019-11-02 01:03 +0000
http://bitbucket.org/pypy/pypy/changeset/48c567ce1caf/

Log:	Add sys._framework attribute

diff --git a/pypy/module/sys/moduledef.py b/pypy/module/sys/moduledef.py
--- a/pypy/module/sys/moduledef.py
+++ b/pypy/module/sys/moduledef.py
@@ -45,6 +45,7 @@
         'argv'                  : 'state.get(space).w_argv',
         'warnoptions'           : 'state.get(space).w_warnoptions',
         'abiflags'              : 'space.wrap("")',
+        '_framework': "space.newtext('')",
         'builtin_module_names'  : 'space.w_None',
         'pypy_getudir'          : 'state.pypy_getudir',    # not translated
         'pypy_find_stdlib'      : 'initpath.pypy_find_stdlib',
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -75,6 +75,7 @@
         builtin2 = sys.modules['builtins']
         assert builtins is builtin2, ( "import builtins "
                                        "is not sys.modules[builtins].")
+
     def test_builtin_module_names(self):
         import sys
         names = sys.builtin_module_names
@@ -163,7 +164,6 @@
             # see comment in 'setup_after_space_initialization'
             untranslated_enc = {'win32': 'mbcs', 'darwin': 'utf-8'}.get(enc, 'ascii')
             assert enc == untranslated_enc
-            
 
     def test_float_info(self):
         import sys
@@ -635,6 +635,7 @@
         assert isinstance(vi[2], int)
         assert vi[3] in ("alpha", "beta", "candidate", "final")
         assert isinstance(vi[4], int)
+        assert isinstance(sys._framework, str)
 
     def test_implementation(self):
         import sys

From pypy.commits at gmail.com  Sat Nov  2 09:58:00 2019
From: pypy.commits at gmail.com (bern...@inf.ethz.ch)
Date: Sat, 02 Nov 2019 06:58:00 -0700 (PDT)
Subject: [pypy-commit] pypy py3.7: Implementing isascii for bytes and
 bytesarray (see bpo-32677)
Message-ID: <5dbd8b68.1c69fb81.6b5e7.6c83@mx.google.com>

Author: bernd.schoeller at inf.ethz.ch
Branch: py3.7
Changeset: r97938:04142a3f29de
Date: 2019-11-02 13:50 +0000
http://bitbucket.org/pypy/pypy/changeset/04142a3f29de/

Log:	Implementing isascii for bytes and bytesarray (see bpo-32677)

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -355,6 +355,12 @@
             return space.w_NotImplemented
         return space.newbool(cmp > 0 or (cmp == 0 and self._len() >= other_len))
 
+    def descr_isascii(self, space):
+        for i in self._data[self._offset:]:
+            if ord(i) > 127:
+                return space.w_False
+        return space.w_True
+
     def descr_inplace_add(self, space, w_other):
         if isinstance(w_other, W_BytearrayObject):
             self._data += w_other.getdata()
@@ -815,6 +821,12 @@
         and there is at least one character in B, False otherwise.
         """
 
+    def isascii():
+        """B.isascii() -> bool
+
+        Return true if the string is empty or all characters in the string are ASCII, false otherwise.
+        ASCII characters have code points in the range U+0000-U+007F."""
+
     def isdigit():
         """B.isdigit() -> bool
 
@@ -1112,6 +1124,8 @@
                          doc=BytearrayDocstrings.isalnum.__doc__),
     isalpha = interp2app(W_BytearrayObject.descr_isalpha,
                          doc=BytearrayDocstrings.isalpha.__doc__),
+    isascii = interp2app(W_BytearrayObject.descr_isascii,
+                         doc=BytearrayDocstrings.isascii.__doc__),
     isdigit = interp2app(W_BytearrayObject.descr_isdigit,
                          doc=BytearrayDocstrings.isdigit.__doc__),
     islower = interp2app(W_BytearrayObject.descr_islower,
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -73,6 +73,12 @@
     def descr_hash(self, space):
         """x.__hash__() <==> hash(x)"""
 
+    def descr_isascii(self, space):
+        """B.isascii() -> bool
+
+        Return true if the string is empty or all characters in the string are ASCII, false otherwise.
+        ASCII characters have code points in the range U+0000-U+007F."""
+
     def descr_iter(self, space):
         """x.__iter__() <==> iter(x)"""
 
@@ -601,6 +607,12 @@
         x -= (x == -1) # convert -1 to -2 without creating a bridge
         return space.newint(x)
 
+    def descr_isascii(self, space):
+        for i in self._value:
+            if ord(i) > 127:
+                return space.w_False
+        return space.w_True
+
     def descr_eq(self, space, w_other):
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
@@ -862,6 +874,7 @@
     isspace = interpindirect2app(W_AbstractBytesObject.descr_isspace),
     istitle = interpindirect2app(W_AbstractBytesObject.descr_istitle),
     isupper = interpindirect2app(W_AbstractBytesObject.descr_isupper),
+    isascii = interpindirect2app(W_AbstractBytesObject.descr_isascii),
     join = interpindirect2app(W_AbstractBytesObject.descr_join),
     ljust = interpindirect2app(W_AbstractBytesObject.descr_ljust),
     rjust = interpindirect2app(W_AbstractBytesObject.descr_rjust),
diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py
--- a/pypy/objspace/std/test/test_bytearrayobject.py
+++ b/pypy/objspace/std/test/test_bytearrayobject.py
@@ -605,6 +605,17 @@
     def test_hex(self):
         assert bytearray(b'santa claus').hex() == "73616e746120636c617573"
 
+    def test_isascii(self):
+        assert bytearray(b'hello world').isascii() is True
+        assert bytearray(b'\x00\x7f').isascii() is True
+        assert bytearray(b'\x80').isascii() is False
+        ba = bytearray(b"\xff\xffHello World")
+        assert ba.isascii() is False
+        del ba[0:1]
+        assert ba.isascii() is False
+        del ba[0:1]
+        assert ba.isascii() is True
+
     def test_format(self):
         """
         assert bytearray(b'a%db') % 2 == b'a2b'
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -281,6 +281,15 @@
         assert b'aaaa'.capitalize() == b'Aaaa'
         assert b'AaAa'.capitalize() == b'Aaaa'
 
+    def test_isascii(self):
+        assert b"hello".isascii() is True
+        assert b"\x00\x7f".isascii() is True
+        assert b"\x80".isascii() is False
+        assert b"\x97".isascii() is False
+        assert b"\xff".isascii() is False
+        assert b"Hello World\x00".isascii() is True
+        assert b"Hello World\x80".isascii() is False
+
     def test_rjust(self):
         s = b"abc"
         assert s.rjust(2) == s

From pypy.commits at gmail.com  Sat Nov  2 11:18:39 2019
From: pypy.commits at gmail.com (schoelle)
Date: Sat, 02 Nov 2019 08:18:39 -0700 (PDT)
Subject: [pypy-commit] pypy py3.7: Made fromhex ignore all whitespace (see
 bpo-28927)
Message-ID: <5dbd9e4f.1c69fb81.f5dcd.6006@mx.google.com>

Author: Bernd Schoeller <bernd at fams.de>
Branch: py3.7
Changeset: r97939:dc491b783ef7
Date: 2019-11-02 15:06 +0000
http://bitbucket.org/pypy/pypy/changeset/dc491b783ef7/

Log:	Made fromhex ignore all whitespace (see bpo-28927)

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -556,7 +556,7 @@
     length = len(s)
     i = 0
     while True:
-        while i < length and s[i] == ' ':
+        while i < length and s[i].isspace():
             i += 1
         if i >= length:
             break
diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py
--- a/pypy/objspace/std/test/test_bytearrayobject.py
+++ b/pypy/objspace/std/test/test_bytearrayobject.py
@@ -439,6 +439,8 @@
         b = bytearray([0x1a, 0x2b, 0x30])
         assert bytearray.fromhex('1a2B30') == b
         assert bytearray.fromhex('  1A 2B  30   ') == b
+        assert bytearray.fromhex('''\t1a\t2B\n
+           30\n''') == b
         assert bytearray.fromhex('0000') == b'\0\0'
 
         raises(ValueError, bytearray.fromhex, 'a')
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -137,6 +137,7 @@
         assert bytes.fromhex("abcd") == b'\xab\xcd'
         assert b''.fromhex("abcd") == b'\xab\xcd'
         assert bytes.fromhex("ab cd  ef") == b'\xab\xcd\xef'
+        assert bytes.fromhex("\nab\tcd  \tef\t") == b'\xab\xcd\xef'
         raises(TypeError, bytes.fromhex, b"abcd")
         raises(TypeError, bytes.fromhex, True)
         raises(ValueError, bytes.fromhex, "hello world")

From pypy.commits at gmail.com  Sat Nov  2 12:52:18 2019
From: pypy.commits at gmail.com (schoelle)
Date: Sat, 02 Nov 2019 09:52:18 -0700 (PDT)
Subject: [pypy-commit] pypy py3.7: Checking for overflow in ctypes array
 creation (fixing test.test_ctypes)
Message-ID: <5dbdb442.1c69fb81.9d20d.667e@mx.google.com>

Author: Bernd Schoeller <bernd at fams.de>
Branch: py3.7
Changeset: r97940:9f57693bca37
Date: 2019-11-02 16:45 +0000
http://bitbucket.org/pypy/pypy/changeset/9f57693bca37/

Log:	Checking for overflow in ctypes array creation (fixing
	test.test_ctypes)

diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py
--- a/lib_pypy/_ctypes/array.py
+++ b/lib_pypy/_ctypes/array.py
@@ -275,6 +275,8 @@
         raise TypeError("Can't multiply a ctypes type by a non-integer")
     if length < 0:
         raise ValueError("Array length must be >= 0")
+    if length * base._sizeofinstances() > sys.maxsize:
+        raise OverflowError("array too large")
     key = (base, length)
     try:
         return ARRAY_CACHE[key]

From pypy.commits at gmail.com  Sat Nov  2 13:08:33 2019
From: pypy.commits at gmail.com (arigo)
Date: Sat, 02 Nov 2019 10:08:33 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: hg merge default
Message-ID: <5dbdb811.1c69fb81.37d87.065f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.6
Changeset: r97941:b2dc8d7edf61
Date: 2019-11-02 17:31 +0100
http://bitbucket.org/pypy/pypy/changeset/b2dc8d7edf61/

Log:	hg merge default

diff --git a/lib_pypy/_curses_build.py b/lib_pypy/_curses_build.py
--- a/lib_pypy/_curses_build.py
+++ b/lib_pypy/_curses_build.py
@@ -1,12 +1,18 @@
 from cffi import FFI
 import os
 
+version_str = '''
+    static const int NCURSES_VERSION_MAJOR;
+    static const int NCURSES_VERSION_MINOR;
+'''
+
 def find_library(options):
     for library in options:
         ffi = FFI()
-        ffi.set_source("_curses_cffi_check", "", libraries=[library])
+        ffi.cdef(version_str)
+        ffi.set_source("_curses_cffi_check", version_str, libraries=[library])
         try:
-            ffi.compile()
+            lib = ffi.compile()
         except VerificationError as e:
             e_last = e
             continue
@@ -21,12 +27,11 @@
     if os.path.exists('/usr/include/ncurses'):
         return ['/usr/include/ncurses']
     if os.path.exists('/usr/include/ncursesw'):
-    	return ['/usr/include/ncursesw']
+        return ['/usr/include/ncursesw']
     return []
 
 
 ffi = FFI()
-
 ffi.set_source("_curses_cffi", """
 #ifdef __APPLE__
 /* the following define is necessary for OS X 10.6+; without it, the
@@ -77,6 +82,9 @@
                 find_library(['panel', 'panelw'])],
      include_dirs=find_curses_include_dirs())
 
+import _curses_cffi_check
+lib = _curses_cffi_check.lib
+version = (lib.NCURSES_VERSION_MAJOR, lib.NCURSES_VERSION_MINOR)
 
 ffi.cdef("""
 typedef ... WINDOW;
@@ -186,8 +194,6 @@
 void filter(void);
 int flash(void);
 int flushinp(void);
-int wget_wch(WINDOW *, wint_t *);
-int mvwget_wch(WINDOW *, int, int, wint_t *);
 chtype getbkgd(WINDOW *);
 WINDOW * getwin(FILE *);
 int halfdelay(int);
@@ -263,7 +269,6 @@
 int touchwin(WINDOW *);
 int typeahead(int);
 int ungetch(int);
-int unget_wch(const wchar_t);
 int untouchwin(WINDOW *);
 void use_env(bool);
 int waddch(WINDOW *, const chtype);
@@ -367,6 +372,13 @@
 void _m_getsyx(int *yx);
 """)
 
+if version > (5, 7):
+    ffi.cdef("""
+int wget_wch(WINDOW *, wint_t *);
+int mvwget_wch(WINDOW *, int, int, wint_t *);
+int unget_wch(const wchar_t);
+""")
+
 
 if __name__ == "__main__":
     ffi.compile()
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -348,6 +348,9 @@
             return space.w_None
         return tb
 
+    def got_any_traceback(self):
+        return self._application_traceback is not None
+
     def set_traceback(self, traceback):
         """Set the current traceback."""
         self._application_traceback = traceback
diff --git a/pypy/interpreter/test/test_class.py b/pypy/interpreter/test/test_class.py
--- a/pypy/interpreter/test/test_class.py
+++ b/pypy/interpreter/test/test_class.py
@@ -123,3 +123,20 @@
         assert C.__qualname__ == 'test_qualname.<locals>.C'
         assert C.D.__qualname__ == 'test_qualname.<locals>.C.D'
         assert not hasattr(C(), '__qualname__')
+
+    def test_nonsensical_base_error_message(self):
+        with raises(TypeError) as exc:
+            class Foo('base'):
+                pass
+        assert str(exc.value).startswith(
+            "metaclass found to be 'str', but calling <class 'str'> "
+            "with args ('Foo', ('base',), ...) raised ")
+        #
+        def foo_func(): pass
+        with raises(TypeError) as exc:
+            class Foo(foo_func):
+                pass
+        assert str(exc.value).startswith(
+            "metaclass found to be 'function', but calling <class 'function'> "
+            "with args ('Foo', (<function test_nonsensical_base_error_message"
+            ".<locals>.foo_func at ")
diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -149,7 +149,19 @@
                      args_w=[w_name, w_bases, w_namespace],
                      keywords=keywords,
                      keywords_w=kwds_w.values())
-    w_class = space.call_args(w_meta, args)
+    try:
+        w_class = space.call_args(w_meta, args)
+    except OperationError as e:
+        # give a more comprehensible error message for TypeErrors
+        if e.got_any_traceback():
+            raise
+        if not e.match(space, space.w_TypeError):
+            raise
+        raise oefmt(space.w_TypeError,
+            "metaclass found to be '%N', but calling %R "
+            "with args (%R, %R, ...) raised %R",
+            w_meta, w_meta, w_name, w_bases,
+            e.get_w_value(space))
     if isinstance(w_cell, Cell) and isinstance(w_class, W_TypeObject):
         if w_cell.empty():
             # will become an error in Python 3.7
diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -343,7 +343,14 @@
         currmap = self.startmap
         while True:
             # parse a key: value
-            currmap = self.decode_key_map(i, currmap)
+            newmap = self.decode_key_map(i, currmap)
+            if newmap is None:
+                # We've seen a repeated value, switch to dict-based storage.
+                dict_w = self._switch_to_dict(currmap, values_w, nextindex)
+                # We re-parse the last key, to get the correct overwriting
+                # effect. Pointless to care for performance here.
+                return self.decode_object_dict(i, start, dict_w)
+            currmap = newmap
             i = self.skip_whitespace(self.pos)
             ch = self.ll_chars[i]
             if ch != ':':
@@ -611,6 +618,8 @@
         """ Given the current map currmap of an object, decode the next key at
         position i. This returns the new map of the object. """
         newmap = self._decode_key_map(i, currmap)
+        if newmap is None:
+            return None
         currmap.observe_transition(newmap, self.startmap)
         return newmap
 
@@ -790,6 +799,11 @@
             self.nextmap_first._check_invariants()
 
     def get_next(self, w_key, string, start, stop, terminator):
+        """ Returns the next map, given a wrapped key w_key, the json input
+        string with positions start and stop, as well as a terminator.
+
+        Returns None if the key already appears somewhere in the map chain.
+        """
         from pypy.objspace.std.dictmultiobject import unicode_hash, unicode_eq
         if isinstance(self, JSONMap):
             assert not self.state == MapBase.BLOCKED
@@ -804,6 +818,8 @@
         if nextmap_first is None:
             # first transition ever seen, don't initialize nextmap_all
             next = self._make_next_map(w_key, string[start:stop])
+            if next is None:
+                return None
             self.nextmap_first = next
         else:
             if self.nextmap_all is None:
@@ -818,6 +834,8 @@
             # if we are at this point we didn't find the transition yet, so
             # create a new one
             next = self._make_next_map(w_key, string[start:stop])
+            if next is None:
+                return None
             self.nextmap_all[w_key] = next
 
             # one new leaf has been created
@@ -860,6 +878,14 @@
                 self.mark_useful(terminator)
 
     def _make_next_map(self, w_key, key_repr):
+        # Check whether w_key is already part of the self.prev chain
+        # to prevent strangeness in the json dict implementation.
+        # This is slow, but it should be rare to call this function.
+        check = self
+        while isinstance(check, JSONMap):
+            if check.w_key._utf8 == w_key._utf8:
+                return None
+            check = check.prev
         return JSONMap(self.space, self, w_key, key_repr)
 
     def fill_dict(self, dict_w, values_w):
diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py
--- a/pypy/module/_pypyjson/test/test__pypyjson.py
+++ b/pypy/module/_pypyjson/test/test__pypyjson.py
@@ -74,6 +74,17 @@
         m3.fill_dict(d, [space.w_None, space.w_None, space.w_None])
         assert list(d) == [w_a, w_b, w_c]
 
+    def test_repeated_key_get_next(self):
+        m = Terminator(self.space)
+        w_a = self.space.newutf8("a", 1)
+        w_b = self.space.newutf8("b", 1)
+        w_c = self.space.newutf8("c", 1)
+        m1 = m.get_next(w_a, '"a"', 0, 3, m)
+        m1 = m1.get_next(w_b, '"b"', 0, 3, m)
+        m1 = m1.get_next(w_c, '"c"', 0, 3, m)
+        m2 = m1.get_next(w_a, '"a"', 0, 3, m)
+        assert m2 is None
+
 
     def test_decode_key_map(self):
         m = Terminator(self.space)
@@ -531,3 +542,12 @@
             pass
         exc = raises(MyError, _pypyjson.loads, 'nul', MyError)
         assert exc.value.args == ('Error when decoding null', 'nul', 1)
+
+    def test_repeated_key(self):
+        import _pypyjson
+        a = '{"abc": "4", "k": 1, "k": 2}'
+        d = _pypyjson.loads(a)
+        assert d == {u"abc": u"4", u"k": 2}
+        a = '{"abc": "4", "k": 1, "k": 1.5, "c": null, "k": 2}'
+        d = _pypyjson.loads(a)
+        assert d == {u"abc": u"4", u"c": None, u"k": 2}
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -1251,7 +1251,8 @@
         except TypeError as e:
             import sys
             if '__pypy__' in sys.builtin_module_names:
-                assert str(e) == 'instance layout conflicts in multiple inheritance'
+                print(str(e))
+                assert 'instance layout conflicts in multiple inheritance' in str(e)
 
             else:
                 assert 'instance lay-out conflict' in str(e)
diff --git a/pypy/objspace/std/test/test_jsondict.py b/pypy/objspace/std/test/test_jsondict.py
--- a/pypy/objspace/std/test/test_jsondict.py
+++ b/pypy/objspace/std/test/test_jsondict.py
@@ -87,3 +87,19 @@
         assert not 1 in d
         assert __pypy__.strategy(d) == "UnicodeDictStrategy"
         assert list(d) == [u"a", u"b"]
+
+    def test_bug(self):
+        import _pypyjson
+        a =  """
+        {
+          "top": {
+            "k": "8",
+            "k": "8",
+            "boom": 1
+          }
+        }
+        """
+        d = _pypyjson.loads(a)
+        str(d)
+        repr(d)
+
diff --git a/pypy/tool/release/make_portable.py b/pypy/tool/release/make_portable.py
new file mode 100644
--- /dev/null
+++ b/pypy/tool/release/make_portable.py
@@ -0,0 +1,110 @@
+#!/usr/bin/env python
+
+bundle = ['sqlite3', 'ssl', 'crypto', 'ffi', 'expat', 'tcl', 'tk', 'gdbm',
+          'lzma', 'tinfo', 'tinfow', 'ncursesw', 'panelw', 'ncurses', 'panel', 'panelw']
+
+from os import chdir, mkdir, symlink
+from os.path import dirname, relpath, join, exists, basename, realpath
+from shutil import copy2
+import sys
+from glob import glob
+from subprocess import check_output, check_call
+
+
+def get_deps(binary):
+    deps = {}
+    output = check_output(['ldd', binary])
+    for line in output.splitlines():
+        if '=>' not in line:
+            continue
+        line = line.strip()
+        needed, path = line.split(' => ')
+        if path == 'not found':
+            print('Broken dependency in ' + binary)
+        path = path.split(' ')[0]
+        path = realpath(path)
+        if not path:
+            continue
+
+        if needed[3:].split('.', 1)[0] not in bundle:
+            continue
+
+        deps[needed] = path
+        deps.update(get_deps(path))
+
+    return deps
+
+
+def gather_deps(binaries):
+    deps = {}
+    for binary in binaries:
+        deps.update(get_deps(binary))
+
+    return deps
+
+
+def copy_deps(deps):
+    copied = {}
+
+    for needed, path in deps.items():
+        bname = basename(path)
+
+        copy2(path, 'lib/' + bname)
+        copied[path] = 'lib/' + bname
+
+        if not exists('lib/' + needed):
+            symlink(bname, 'lib/' + needed)
+
+    return copied
+
+
+def rpath_binaries(binaries):
+    rpaths = {}
+
+    for binary in binaries:
+        rpath = join('$ORIGIN', relpath('lib', dirname(binary)))
+        check_call(['patchelf', '--set-rpath', rpath, binary])
+
+        rpaths[binary] = rpath
+
+    return rpaths
+
+
+def make_portable():
+    binaries = glob('bin/libpypy*.so')
+    if not binaries:
+        raise ValueError('Could not find bin/libpypy*.so')
+    binaries.extend(glob('lib_pypy/*_cffi.pypy*.so'))
+    binaries.extend(glob('lib_pypy/_pypy_openssl*.so'))
+    binaries.extend(glob('lib_pypy/_tkinter/*_cffi.pypy*.so'))
+
+    deps = gather_deps(binaries)
+
+    copied = copy_deps(deps)
+
+    for path, item in copied.items():
+        print('Copied {0} to {1}'.format(path, item))
+
+    binaries.extend(copied.values())
+
+    rpaths = rpath_binaries(binaries)
+    for binary, rpath in rpaths.items():
+        print('Set RPATH of {0} to {1}'.format(binary, rpath))
+
+    return deps
+
+
+if __name__ == '__main__':
+    try:
+        chdir(sys.argv[1])
+    except:
+        print('Call as %s <path/to/pypy/topdir' % sys.argv[0])
+        exit(-1)
+
+    try:
+        mkdir('lib')
+    except OSError:
+        pass
+
+    make_portable()
+
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+from __future__ import print_function 
 """ packages PyPy, provided that it's already built.
 It uses 'pypy/goal/pypy3-c' and parts of the rest of the working
 copy.  Usage:
@@ -20,7 +21,9 @@
 import py
 import fnmatch
 import subprocess
+import platform
 from pypy.tool.release.smartstrip import smartstrip
+from pypy.tool.release.make_portable import make_portable
 
 USE_ZIPFILE_MODULE = sys.platform == 'win32'
 
@@ -90,10 +93,11 @@
         )
 
         for key, module in failures:
-            print >>sys.stderr, """!!!!!!!!!!\nBuilding {0} bindings failed.
+            print("""!!!!!!!!!!\nBuilding {0} bindings failed.
                 You can either install development headers package,
                 add the --without-{0} option to skip packaging this
-                binary CFFI extension, or say --without-cffi.""".format(key)
+                binary CFFI extension, or say --without-cffi.""".format(key),
+                file=sys.stderr)
         if len(failures) > 0:
             return 1, None
 
@@ -130,7 +134,7 @@
         if pypyw.exists():
             tgt = py.path.local(tgt)
             binaries.append((pypyw, tgt.new(purebasename=tgt.purebasename + 'w').basename, None))
-            print "Picking %s" % str(pypyw)
+            print("Picking %s" % str(pypyw))
         # Can't rename a DLL: it is always called 'libpypy3-c.dll'
         win_extras = [('libpypy3-c.dll', None), ('sqlite3.dll', lib_pypy)]
         if not options.no_tk:
@@ -142,18 +146,19 @@
             if not p.check():
                 p = py.path.local.sysfind(extra)
                 if not p:
-                    print "%s not found, expect trouble if this is a shared build" % (extra,)
+                    print("%s not found, expect trouble if this "
+                          "is a shared build" % (extra,))
                     continue
-            print "Picking %s" % p
+            print("Picking %s" % p)
             binaries.append((p, p.basename, target_dir))
         libsdir = basedir.join('libs')
         if libsdir.exists():
-            print 'Picking %s (and contents)' % libsdir
+            print('Picking %s (and contents)' % libsdir)
             shutil.copytree(str(libsdir), str(pypydir.join('libs')))
         else:
-            print '"libs" dir with import library not found.'
-            print 'You have to create %r' % (str(libsdir),)
-            print 'and copy libpypy3-c.lib in there, renamed to python32.lib'
+            print('"libs" dir with import library not found.')
+            print('You have to create %r' % (str(libsdir),))
+            print('and copy libpypy3-c.lib in there, renamed to python32.lib')
             # XXX users will complain that they cannot compile capi (cpyext)
             # modules for windows, also embedding pypy (i.e. in cffi)
             # will fail.
@@ -170,14 +175,15 @@
                 tktcldir = p.dirpath().join('..').join('lib')
                 shutil.copytree(str(tktcldir), str(pypydir.join('tcl')))
             except WindowsError:
-                print >>sys.stderr, r"""Packaging Tk runtime failed.
-tk85.dll and tcl85.dll found in %s, expecting to find runtime in %s
-directory next to the dlls, as per build instructions.""" %(p, tktcldir)
+                print("Packaging Tk runtime failed. tk85.dll and tcl85.dll "
+                      "found in %s, expecting to find runtime in %s directory "
+                      "next to the dlls, as per build "
+                      "instructions." %(p, tktcldir), file=sys.stderr)
                 import traceback;traceback.print_exc()
                 raise MissingDependenciesError('Tk runtime')
 
-    print '* Binaries:', [source.relto(str(basedir))
-                          for source, target, target_dir in binaries]
+    print('* Binaries:', [source.relto(str(basedir))
+                          for source, target, target_dir in binaries])
 
     # Careful: to copy lib_pypy, copying just the hg-tracked files
     # would not be enough: there are also ctypes_config_cache/_*_cache.py.
@@ -238,7 +244,11 @@
                 else:
                     archive = bindir.join(target)
                 smartstrip(archive, keep_debug=options.keep_debug)
-        #
+
+            # make the package portable by adding rpath=$ORIGIN/..lib,
+            # bundling dependencies
+            if options.make_portable:
+                make_portable()
         if USE_ZIPFILE_MODULE:
             import zipfile
             archive = str(builddir.join(name + '.zip'))
@@ -252,10 +262,11 @@
         else:
             archive = str(builddir.join(name + '.tar.bz2'))
             if sys.platform == 'darwin':
-                print >>sys.stderr, """Warning: tar on current platform does not suport overriding the uid and gid
-for its contents. The tarball will contain your uid and gid. If you are
-building the actual release for the PyPy website, you may want to be
-using another platform..."""
+                print("Warning: tar on current platform does not suport "
+                      "overriding the uid and gid for its contents. The tarball "
+                      "will contain your uid and gid. If you are building the "
+                      "actual release for the PyPy website, you may want to be "
+                      "using another platform...", file=sys.stderr)
                 e = os.system('tar --numeric-owner -cvjf ' + archive + " " + name)
             elif sys.platform.startswith('freebsd'):
                 e = os.system('tar --uname=root --gname=wheel -cvjf ' + archive + " " + name)
@@ -268,10 +279,10 @@
     finally:
         os.chdir(old_dir)
     if options.targetdir:
-        print "Copying %s to %s" % (archive, options.targetdir)
+        print("Copying %s to %s" % (archive, options.targetdir))
         shutil.copy(archive, options.targetdir)
     else:
-        print "Ready in %s" % (builddir,)
+        print("Ready in %s" % (builddir,))
     return retval, builddir # for tests
 
 def package(*args, **kwds):
@@ -319,8 +330,14 @@
                         dest='embed_dependencies',
                         action=NegateAction,
                         default=(sys.platform == 'darwin'),
-                        help='whether to embed dependencies for distribution '
+                        help='whether to embed dependencies in CFFI modules '
                         '(default on OS X)')
+    parser.add_argument('--make-portable', '--no-make-portable',
+                        dest='make_portable',
+                        action=NegateAction,
+                        default=(platform.linux_distribution() in ('CentOS',)),
+                        help='whether to make the package portable by shipping '
+                            'dependent shared objects and mangling RPATH')
     options = parser.parse_args(args)
 
     if os.environ.has_key("PYPY_PACKAGE_NOKEEPDEBUG"):
@@ -331,6 +348,10 @@
         options.embed_dependencies = True
     elif os.environ.has_key("PYPY_NO_EMBED_DEPENDENCIES"):
         options.embed_dependencies = False
+    if os.environ.has_key("PYPY_MAKE_PORTABLE"):
+        options.embed_dependencies = True
+    elif os.environ.has_key("PYPY_NO_MAKE_PORTABLE"):
+        options.embed_dependencies = False
     if not options.builddir:
         # The import actually creates the udir directory
         from rpython.tool.udir import udir

From pypy.commits at gmail.com  Sun Nov  3 01:37:23 2019
From: pypy.commits at gmail.com (arigo)
Date: Sat, 02 Nov 2019 23:37:23 -0700 (PDT)
Subject: [pypy-commit] cffi default: Tweak the warning message
Message-ID: <5dbe75a3.1c69fb81.e1b0.8a8c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3308:0d43d8bf557f
Date: 2019-11-03 07:30 +0100
http://bitbucket.org/cffi/cffi/changeset/0d43d8bf557f/

Log:	Tweak the warning message

diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -159,9 +159,9 @@
 def _warn_for_non_extern_non_static_global_variable(decl):
     if not decl.storage:
         import warnings
-        warnings.warn("Declaration of global variable '%s' in cdef() should "
-                      "be marked 'extern' for consistency (or possibly "
-                      "'static' in API mode)" % (decl.name,))
+        warnings.warn("Global variable '%s' in cdef(): for consistency "
+                      "with C it should have a storage class specifier "
+                      "(usually 'extern')" % (decl.name,))
 
 def _preprocess(csource):
     # Remove comments.  NOTE: this only work because the cdef() section

From pypy.commits at gmail.com  Sun Nov  3 01:37:25 2019
From: pypy.commits at gmail.com (arigo)
Date: Sat, 02 Nov 2019 23:37:25 -0700 (PDT)
Subject: [pypy-commit] cffi release-1.13: hg merge default
Message-ID: <5dbe75a5.1c69fb81.d80ef.84c3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-1.13
Changeset: r3309:b175e63b614b
Date: 2019-11-03 07:30 +0100
http://bitbucket.org/cffi/cffi/changeset/b175e63b614b/

Log:	hg merge default

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -22,3 +22,4 @@
 e0c76668ecf5ab1ad005799cfe1b7cb58610f155 v1.12.2
 bf80722dea36237710083315e336c81bc8571fd0 v1.12.3
 17c06e75f54cf5a31865961f0b498f99232a0bc8 v1.13.0
+9cc790d05bc5256eee32fddbe60aadf24f14ce44 v1.13.1
diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -159,9 +159,9 @@
 def _warn_for_non_extern_non_static_global_variable(decl):
     if not decl.storage:
         import warnings
-        warnings.warn("Declaration of global variable '%s' in cdef() should "
-                      "be marked 'extern' for consistency (or possibly "
-                      "'static' in API mode)" % (decl.name,))
+        warnings.warn("Global variable '%s' in cdef(): for consistency "
+                      "with C it should have a storage class specifier "
+                      "(usually 'extern')" % (decl.name,))
 
 def _preprocess(csource):
     # Remove comments.  NOTE: this only work because the cdef() section
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -54,11 +54,11 @@
 
 * Checksums of the "source" package version 1.13.1:
 
-   - MD5: ...
+   - MD5: 824ad9f228fbc6ce203c334e2ff4ab8f
 
-   - SHA: ...
+   - SHA: 678dad3a3102ebcbeab8f5f132968afa444f6a54
 
-   - SHA256: ...
+   - SHA256: 558b3afef987cf4b17abd849e7bedf64ee12b28175d564d05b628a0f9355599b
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``

From pypy.commits at gmail.com  Sun Nov  3 01:37:27 2019
From: pypy.commits at gmail.com (arigo)
Date: Sat, 02 Nov 2019 23:37:27 -0700 (PDT)
Subject: [pypy-commit] cffi release-1.13: update version to 1.13.2
Message-ID: <5dbe75a7.1c69fb81.9d20d.f84b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-1.13
Changeset: r3310:33354a4abeb4
Date: 2019-11-03 07:33 +0100
http://bitbucket.org/cffi/cffi/changeset/33354a4abeb4/

Log:	update version to 1.13.2

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -2,7 +2,7 @@
 #include <Python.h>
 #include "structmember.h"
 
-#define CFFI_VERSION  "1.13.1"
+#define CFFI_VERSION  "1.13.2"
 
 #ifdef MS_WIN32
 #include <windows.h>
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -14,7 +14,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.13.1", ("This test_c.py file is for testing a version"
+assert __version__ == "1.13.2", ("This test_c.py file is for testing a version"
                                  " of cffi that differs from the one that we"
                                  " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
diff --git a/cffi/__init__.py b/cffi/__init__.py
--- a/cffi/__init__.py
+++ b/cffi/__init__.py
@@ -5,8 +5,8 @@
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 from .error import PkgConfigError
 
-__version__ = "1.13.1"
-__version_info__ = (1, 13, 1)
+__version__ = "1.13.2"
+__version_info__ = (1, 13, 2)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/cffi/_embedding.h b/cffi/_embedding.h
--- a/cffi/_embedding.h
+++ b/cffi/_embedding.h
@@ -224,7 +224,7 @@
 
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.13.1"
+                               "\ncompiled with cffi version: 1.13.2"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/doc/source/conf.py b/doc/source/conf.py
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -47,7 +47,7 @@
 # The short X.Y version.
 version = '1.13'
 # The full version, including alpha/beta/rc tags.
-release = '1.13.1'
+release = '1.13.2'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -52,13 +52,13 @@
 
 * https://pypi.python.org/pypi/cffi
 
-* Checksums of the "source" package version 1.13.1:
+* Checksums of the "source" package version 1.13.2:
 
-   - MD5: 824ad9f228fbc6ce203c334e2ff4ab8f
+   - MD5: ...
 
-   - SHA: 678dad3a3102ebcbeab8f5f132968afa444f6a54
+   - SHA: ...
 
-   - SHA256: 558b3afef987cf4b17abd849e7bedf64ee12b28175d564d05b628a0f9355599b
+   - SHA256: ...
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -198,7 +198,7 @@
 
 `Mailing list <https://groups.google.com/forum/#!forum/python-cffi>`_
 """,
-        version='1.13.1',
+        version='1.13.2',
         packages=['cffi'] if cpython else [],
         package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h', 
                                '_embedding.h', '_cffi_errors.h']}

From pypy.commits at gmail.com  Sun Nov  3 01:37:28 2019
From: pypy.commits at gmail.com (arigo)
Date: Sat, 02 Nov 2019 23:37:28 -0700 (PDT)
Subject: [pypy-commit] cffi release-1.13: update whatsnew
Message-ID: <5dbe75a8.1c69fb81.496da.b149@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-1.13
Changeset: r3311:31d60585c288
Date: 2019-11-03 07:36 +0100
http://bitbucket.org/cffi/cffi/changeset/31d60585c288/

Log:	update whatsnew

diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -2,6 +2,15 @@
 What's New
 ======================
 
+v1.13.2
+=======
+
+* re-release because the Linux wheels came with an attached version of libffi
+  that was very old and buggy (`issue #432`_).
+
+.. _`issue #432`: https://bitbucket.org/cffi/cffi/issues/432/
+
+
 
 v1.13.1
 =======

From pypy.commits at gmail.com  Sun Nov  3 06:13:41 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 03 Nov 2019 03:13:41 -0800 (PST)
Subject: [pypy-commit] pypy default: refactor "os import" in
 make_portable.py to make it importable on windows
Message-ID: <5dbeb665.1c69fb81.db315.5e13@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r97942:9f7ce234ad3a
Date: 2019-11-02 05:58 -0400
http://bitbucket.org/pypy/pypy/changeset/9f7ce234ad3a/

Log:	refactor "os import" in make_portable.py to make it importable on
	windows

diff --git a/pypy/tool/release/make_portable.py b/pypy/tool/release/make_portable.py
--- a/pypy/tool/release/make_portable.py
+++ b/pypy/tool/release/make_portable.py
@@ -3,7 +3,7 @@
 bundle = ['sqlite3', 'ssl', 'crypto', 'ffi', 'expat', 'tcl', 'tk', 'gdbm',
           'lzma', 'tinfo', 'tinfow', 'ncursesw', 'panelw', 'ncurses', 'panel', 'panelw']
 
-from os import chdir, mkdir, symlink
+import os
 from os.path import dirname, relpath, join, exists, basename, realpath
 from shutil import copy2
 import sys
@@ -53,7 +53,7 @@
         copied[path] = 'lib/' + bname
 
         if not exists('lib/' + needed):
-            symlink(bname, 'lib/' + needed)
+            os.symlink(bname, 'lib/' + needed)
 
     return copied
 
@@ -96,13 +96,13 @@
 
 if __name__ == '__main__':
     try:
-        chdir(sys.argv[1])
+        os.chdir(sys.argv[1])
     except:
         print('Call as %s <path/to/pypy/topdir' % sys.argv[0])
         exit(-1)
 
     try:
-        mkdir('lib')
+        os.mkdir('lib')
     except OSError:
         pass
 

From pypy.commits at gmail.com  Sun Nov  3 06:13:43 2019
From: pypy.commits at gmail.com (schoelle)
Date: Sun, 03 Nov 2019 03:13:43 -0800 (PST)
Subject: [pypy-commit] pypy default: Checking for overflow in ctypes array
 creation (fixing test.test_ctypes)
Message-ID: <5dbeb667.1c69fb81.2fa61.6c21@mx.google.com>

Author: Bernd Schoeller <bernd at fams.de>
Branch: 
Changeset: r97943:f67d306a0f3a
Date: 2019-11-02 16:45 +0000
http://bitbucket.org/pypy/pypy/changeset/f67d306a0f3a/

Log:	Checking for overflow in ctypes array creation (fixing
	test.test_ctypes) (grafted from
	9f57693bca378be44fe87174966518b7d9799be6)

diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py
--- a/lib_pypy/_ctypes/array.py
+++ b/lib_pypy/_ctypes/array.py
@@ -267,6 +267,8 @@
         raise TypeError("Can't multiply a ctypes type by a non-integer")
     if length < 0:
         raise ValueError("Array length must be >= 0")
+    if length * base._sizeofinstances() > sys.maxsize:
+        raise OverflowError("array too large")
     key = (base, length)
     try:
         return ARRAY_CACHE[key]

From pypy.commits at gmail.com  Sun Nov  3 06:13:45 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 03 Nov 2019 03:13:45 -0800 (PST)
Subject: [pypy-commit] pypy py3.6: merge default into branch
Message-ID: <5dbeb669.1c69fb81.4cdea.92fb@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r97944:9e6ad1f66385
Date: 2019-11-03 06:12 -0500
http://bitbucket.org/pypy/pypy/changeset/9e6ad1f66385/

Log:	merge default into branch

diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py
--- a/lib_pypy/_ctypes/array.py
+++ b/lib_pypy/_ctypes/array.py
@@ -275,6 +275,8 @@
         raise TypeError("Can't multiply a ctypes type by a non-integer")
     if length < 0:
         raise ValueError("Array length must be >= 0")
+    if length * base._sizeofinstances() > sys.maxsize:
+        raise OverflowError("array too large")
     key = (base, length)
     try:
         return ARRAY_CACHE[key]
diff --git a/pypy/tool/release/make_portable.py b/pypy/tool/release/make_portable.py
--- a/pypy/tool/release/make_portable.py
+++ b/pypy/tool/release/make_portable.py
@@ -3,7 +3,7 @@
 bundle = ['sqlite3', 'ssl', 'crypto', 'ffi', 'expat', 'tcl', 'tk', 'gdbm',
           'lzma', 'tinfo', 'tinfow', 'ncursesw', 'panelw', 'ncurses', 'panel', 'panelw']
 
-from os import chdir, mkdir, symlink
+import os
 from os.path import dirname, relpath, join, exists, basename, realpath
 from shutil import copy2
 import sys
@@ -53,7 +53,7 @@
         copied[path] = 'lib/' + bname
 
         if not exists('lib/' + needed):
-            symlink(bname, 'lib/' + needed)
+            os.symlink(bname, 'lib/' + needed)
 
     return copied
 
@@ -96,13 +96,13 @@
 
 if __name__ == '__main__':
     try:
-        chdir(sys.argv[1])
+        os.chdir(sys.argv[1])
     except:
         print('Call as %s <path/to/pypy/topdir' % sys.argv[0])
         exit(-1)
 
     try:
-        mkdir('lib')
+        os.mkdir('lib')
     except OSError:
         pass
 

From pypy.commits at gmail.com  Sun Nov  3 09:03:47 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Sun, 03 Nov 2019 06:03:47 -0800 (PST)
Subject: [pypy-commit] pypy expose-jsonmap: switch away from the jsondict
 strategy if the map is blocked
Message-ID: <5dbede43.1c69fb81.21629.bde7@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: expose-jsonmap
Changeset: r97945:35f2d7155ef8
Date: 2019-11-03 14:45 +0100
http://bitbucket.org/pypy/pypy/changeset/35f2d7155ef8/

Log:	switch away from the jsondict strategy if the map is blocked

diff --git a/pypy/objspace/std/jsondict.py b/pypy/objspace/std/jsondict.py
--- a/pypy/objspace/std/jsondict.py
+++ b/pypy/objspace/std/jsondict.py
@@ -55,12 +55,18 @@
 
     def is_correct_type(self, w_obj):
         space = self.space
-        return space.is_w(space.type(w_obj), space.w_unicode)
+        # This is a bit of a hack (but probably a good one): when the state of
+        # our jsonmap is blocked, we consider nothing to match type-wise. That
+        # way we use the code paths that switch us to a different strategy.
+        return space.is_w(space.type(w_obj), space.w_unicode) and not self.jsonmap.is_state_blocked()
 
     def _never_equal_to(self, w_lookup_type):
         return False
 
     def length(self, w_dict):
+        if self.jsonmap.is_state_blocked():
+            self.switch_to_unicode_strategy(w_dict)
+            return w_dict.length()
         return len(self.unerase(w_dict.dstorage))
 
     def getitem(self, w_dict, w_key):
@@ -81,6 +87,8 @@
 
     def setitem(self, w_dict, w_key, w_value):
         if self.is_correct_type(w_key):
+            if jit.isconstant(w_key):
+                jit.promote(self)
             storage_w = self.unerase(w_dict.dstorage)
             index = self.jsonmap.get_index(w_key)
             if index != -1:
diff --git a/pypy/objspace/std/test/test_jsondict.py b/pypy/objspace/std/test/test_jsondict.py
--- a/pypy/objspace/std/test/test_jsondict.py
+++ b/pypy/objspace/std/test/test_jsondict.py
@@ -111,3 +111,64 @@
         str(d)
         repr(d)
 
+
+class TestJsonDictBlockedJsonMap(object):
+    def make_jsondict(self):
+        from pypy.module._pypyjson import interp_decoder
+        from pypy.objspace.std.jsondict import from_values_and_jsonmap
+        space = self.space
+        w_a = self.space.newutf8("a", 1)
+        w_b = self.space.newutf8("b", 1)
+        base = interp_decoder.Terminator(space)
+        m1 = base.get_next(w_a, 'a"', 0, 2, base)
+        m2 = m1.get_next(w_b, 'b"', 0, 2, base)
+
+        w_d = from_values_and_jsonmap(space, [w_a, w_b], m2)
+        return base, m2, w_d, w_a, w_b
+
+    def test_getitem(self):
+        space = self.space
+        base, m2, w_d, w_a, w_b = self.make_jsondict()
+        assert space.getitem(w_d, w_a) is w_a
+        assert space.getitem(w_d, w_b) is w_b
+
+        m2.mark_blocked(base)
+        # accessing a dict with a blocked strategy will switch to
+        # UnicodeDictStrategy
+        assert w_d.dstrategy is m2.strategy_instance
+        assert space.getitem(w_d, w_a) is w_a
+        assert space.getitem(w_d, w_b) is w_b
+        assert w_d.dstrategy is not m2.strategy_instance
+
+    def test_setitem(self):
+        space = self.space
+        base, m2, w_d, w_a, w_b = self.make_jsondict()
+        space.setitem(w_d, w_a, w_b)
+        space.setitem(w_d, w_b, w_a)
+
+        m2.mark_blocked(base)
+        assert w_d.dstrategy is m2.strategy_instance
+        space.setitem(w_d, w_a, w_b)
+        space.setitem(w_d, w_b, w_a)
+        assert w_d.dstrategy is not m2.strategy_instance
+
+    def test_len(self):
+        space = self.space
+        base, m2, w_d, w_a, w_b = self.make_jsondict()
+        assert space.len_w(w_d) == 2
+
+        m2.mark_blocked(base)
+        assert w_d.dstrategy is m2.strategy_instance
+        assert space.len_w(w_d) == 2
+        assert w_d.dstrategy is not m2.strategy_instance
+
+    def test_setdefault(self):
+        base, m2, w_d, w_a, w_b = self.make_jsondict()
+        assert w_d.setdefault(w_a, None) is w_a
+        assert w_d.setdefault(w_b, None) is w_b
+
+        m2.mark_blocked(base)
+        assert w_d.dstrategy is m2.strategy_instance
+        assert w_d.setdefault(w_a, None) is w_a
+        assert w_d.setdefault(w_b, None) is w_b
+        assert w_d.dstrategy is not m2.strategy_instance

From pypy.commits at gmail.com  Sun Nov  3 09:03:49 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Sun, 03 Nov 2019 06:03:49 -0800 (PST)
Subject: [pypy-commit] pypy expose-jsonmap: move fields from JsonMap to
 JsonStrategy
Message-ID: <5dbede45.1c69fb81.d4ecc.81e4@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: expose-jsonmap
Changeset: r97946:b23978e0d3fb
Date: 2019-11-03 15:02 +0100
http://bitbucket.org/pypy/pypy/changeset/b23978e0d3fb/

Log:	move fields from JsonMap to JsonStrategy

diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -1017,8 +1017,6 @@
         self.cache_hits = 0
 
         # for jsondict support
-        self.key_to_index = None
-        self.keys_in_order = None
         self.strategy_instance = None
 
     def __repr__(self):
@@ -1139,47 +1137,7 @@
         dict_w[self.w_key] = values_w[index]
         return index + 1
 
-    # _____________________________________________________
-    # methods for JsonDictStrategy
 
-    @jit.elidable
-    def get_index(self, w_key):
-        from pypy.objspace.std.unicodeobject import W_UnicodeObject
-        assert isinstance(w_key, W_UnicodeObject)
-        return self.get_key_to_index().get(w_key, -1)
-
-    def get_key_to_index(self):
-        from pypy.objspace.std.dictmultiobject import unicode_hash, unicode_eq
-        key_to_index = self.key_to_index
-        if key_to_index is None:
-            key_to_index = self.key_to_index = objectmodel.r_dict(unicode_eq, unicode_hash,
-                  force_non_null=True, simple_hash_eq=True)
-            # compute depth
-            curr = self
-            depth = 0
-            while True:
-                depth += 1
-                curr = curr.prev
-                if not isinstance(curr, JSONMap):
-                    break
-
-            curr = self
-            while depth:
-                depth -= 1
-                key_to_index[curr.w_key] = depth
-                curr = curr.prev
-                if not isinstance(curr, JSONMap):
-                    break
-        return key_to_index
-
-    def get_keys_in_order(self):
-        keys_in_order = self.keys_in_order
-        if keys_in_order is None:
-            key_to_index = self.get_key_to_index()
-            keys_in_order = self.keys_in_order = [None] * len(key_to_index)
-            for w_key, index in key_to_index.iteritems():
-                keys_in_order[index] = w_key
-        return keys_in_order
 
     # _____________________________________________________
 
diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py
--- a/pypy/module/_pypyjson/test/test__pypyjson.py
+++ b/pypy/module/_pypyjson/test/test__pypyjson.py
@@ -42,24 +42,6 @@
         assert m4.key_repr == '"c"'
         assert m3.nextmap_first is m4
 
-    def test_json_map_get_index(self):
-        m = Terminator(self.space)
-        w_a = self.space.newutf8("a", 1)
-        w_b = self.space.newutf8("b", 1)
-        w_c = self.space.newutf8("c", 1)
-        m1 = m.get_next(w_a, 'a"', 0, 2, m)
-        assert m1.get_index(w_a) == 0
-        assert m1.get_index(w_b) == -1
-
-        m2 = m.get_next(w_b, 'b"', 0, 2, m)
-        assert m2.get_index(w_b) == 0
-        assert m2.get_index(w_a) == -1
-
-        m3 = m2.get_next(w_c, 'c"', 0, 2, m)
-        assert m3.get_index(w_b) == 0
-        assert m3.get_index(w_c) == 1
-        assert m3.get_index(w_a) == -1
-
     def test_jsonmap_fill_dict(self):
         from collections import OrderedDict
         m = Terminator(self.space)
diff --git a/pypy/objspace/std/jsondict.py b/pypy/objspace/std/jsondict.py
--- a/pypy/objspace/std/jsondict.py
+++ b/pypy/objspace/std/jsondict.py
@@ -11,13 +11,13 @@
 
 
 def from_values_and_jsonmap(space, values_w, jsonmap):
-    if not objectmodel.we_are_translated():
-        assert len(values_w) == len(jsonmap.get_keys_in_order())
-        assert len(values_w) != 0
     debug.make_sure_not_resized(values_w)
     strategy = jsonmap.strategy_instance
     if strategy is None:
         jsonmap.strategy_instance = strategy = JsonDictStrategy(space, jsonmap)
+    if not objectmodel.we_are_translated():
+        assert len(values_w) == len(strategy.get_keys_in_order())
+        assert len(values_w) != 0
     storage = strategy.erase(values_w)
     return W_DictObject(space, strategy, storage)
 
@@ -44,6 +44,9 @@
         DictStrategy.__init__(self, space)
         self.jsonmap = jsonmap
 
+        self.key_to_index = None
+        self.keys_in_order = None
+
     def wrap(self, w_key):
         return w_key
 
@@ -80,7 +83,7 @@
         storage_w = self.unerase(w_dict.dstorage)
         if jit.isconstant(w_key):
             jit.promote(self)
-        index = self.jsonmap.get_index(w_key)
+        index = self.get_index(w_key)
         if index == -1:
             return None
         return storage_w[index]
@@ -90,7 +93,7 @@
             if jit.isconstant(w_key):
                 jit.promote(self)
             storage_w = self.unerase(w_dict.dstorage)
-            index = self.jsonmap.get_index(w_key)
+            index = self.get_index(w_key)
             if index != -1:
                 storage_w[index] = w_value
                 return
@@ -118,7 +121,7 @@
         values_w = self.unerase(w_dict.dstorage)
         storage = strategy.get_empty_storage()
         d_new = strategy.unerase(storage)
-        keys_in_order = self.jsonmap.get_keys_in_order()
+        keys_in_order = self.get_keys_in_order()
         assert len(keys_in_order) == len(values_w)
         for index, w_key in enumerate(keys_in_order):
             assert w_key is not None
@@ -128,7 +131,7 @@
         w_dict.dstorage = storage
 
     def w_keys(self, w_dict):
-        return self.space.newlist(self.jsonmap.get_keys_in_order())
+        return self.space.newlist(self.get_keys_in_order())
 
     def values(self, w_dict):
         return self.unerase(w_dict.dstorage)[:]  # to make resizable
@@ -137,12 +140,12 @@
         space = self.space
         storage_w = self.unerase(w_dict.dstorage)
         res = [None] * len(storage_w)
-        for index, w_key in enumerate(self.jsonmap.get_keys_in_order()):
+        for index, w_key in enumerate(self.get_keys_in_order()):
             res[index] = space.newtuple([w_key, storage_w[index]])
         return res
 
     def getiterkeys(self, w_dict):
-        return iter(self.jsonmap.get_keys_in_order())
+        return iter(self.get_keys_in_order())
 
     def getitervalues(self, w_dict):
         storage_w = self.unerase(w_dict.dstorage)
@@ -150,7 +153,50 @@
 
     def getiteritems_with_hash(self, w_dict):
         storage_w = self.unerase(w_dict.dstorage)
-        return ZipItemsWithHash(self.jsonmap.get_keys_in_order(), storage_w)
+        return ZipItemsWithHash(self.get_keys_in_order(), storage_w)
+
+    # ____________________________________________________________
+    # methods for interpreting the jsonmaps
+
+    @jit.elidable
+    def get_index(self, w_key):
+        from pypy.objspace.std.unicodeobject import W_UnicodeObject
+        assert isinstance(w_key, W_UnicodeObject)
+        return self.get_key_to_index().get(w_key, -1)
+
+    def get_key_to_index(self):
+        from pypy.objspace.std.dictmultiobject import unicode_hash, unicode_eq
+        from pypy.module._pypyjson.interp_decoder import JSONMap
+        key_to_index = self.key_to_index
+        if key_to_index is None:
+            key_to_index = self.key_to_index = objectmodel.r_dict(unicode_eq, unicode_hash,
+                  force_non_null=True, simple_hash_eq=True)
+            # compute depth
+            curr = self.jsonmap
+            depth = 0
+            while True:
+                depth += 1
+                curr = curr.prev
+                if not isinstance(curr, JSONMap):
+                    break
+
+            curr = self.jsonmap
+            while depth:
+                depth -= 1
+                key_to_index[curr.w_key] = depth
+                curr = curr.prev
+                if not isinstance(curr, JSONMap):
+                    break
+        return key_to_index
+
+    def get_keys_in_order(self):
+        keys_in_order = self.keys_in_order
+        if keys_in_order is None:
+            key_to_index = self.get_key_to_index()
+            keys_in_order = self.keys_in_order = [None] * len(key_to_index)
+            for w_key, index in key_to_index.iteritems():
+                keys_in_order[index] = w_key
+        return keys_in_order
 
 
 class ZipItemsWithHash(object):
diff --git a/pypy/objspace/std/test/test_jsondict.py b/pypy/objspace/std/test/test_jsondict.py
--- a/pypy/objspace/std/test/test_jsondict.py
+++ b/pypy/objspace/std/test/test_jsondict.py
@@ -1,3 +1,33 @@
+
+class TestJsonDict(object):
+
+    def test_json_map_get_index(self):
+        from pypy.module._pypyjson import interp_decoder
+        from pypy.objspace.std.jsondict import JsonDictStrategy
+        space = self.space
+
+        m = interp_decoder.Terminator(self.space)
+        w_a = space.newutf8("a", 1)
+        w_b = space.newutf8("b", 1)
+        w_c = space.newutf8("c", 1)
+        m1 = m.get_next(w_a, 'a"', 0, 2, m)
+        dm1 = JsonDictStrategy(space, m1)
+        assert dm1.get_index(w_a) == 0
+        assert dm1.get_index(w_b) == -1
+
+        m2 = m.get_next(w_b, 'b"', 0, 2, m)
+        dm2 = JsonDictStrategy(space, m2)
+        assert dm2.get_index(w_b) == 0
+        assert dm2.get_index(w_a) == -1
+
+        m3 = m2.get_next(w_c, 'c"', 0, 2, m)
+        dm3 = JsonDictStrategy(space, m3)
+        assert dm3.get_index(w_b) == 0
+        assert dm3.get_index(w_c) == 1
+        assert dm3.get_index(w_a) == -1
+
+
+
 
 class AppTest(object):
     spaceconfig = {"objspace.usemodules._pypyjson": True}

From pypy.commits at gmail.com  Sun Nov  3 09:32:29 2019
From: pypy.commits at gmail.com (arigo)
Date: Sun, 03 Nov 2019 06:32:29 -0800 (PST)
Subject: [pypy-commit] cffi release-1.13: md5/sha
Message-ID: <5dbee4fd.1c69fb81.97af9.100e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-1.13
Changeset: r3312:0c9be4c0d798
Date: 2019-11-03 15:29 +0100
http://bitbucket.org/cffi/cffi/changeset/0c9be4c0d798/

Log:	md5/sha

diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -54,11 +54,11 @@
 
 * Checksums of the "source" package version 1.13.2:
 
-   - MD5: ...
+   - MD5: 652203cf99faa254efff7fab23c2f3a2
 
-   - SHA: ...
+   - SHA: ad94c1b64570ec3016a945d4b226d527de81f9b9
 
-   - SHA256: ...
+   - SHA256: 599a1e8ff057ac530c9ad1778293c665cb81a791421f46922d80a86473c13346
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``

From pypy.commits at gmail.com  Sun Nov  3 09:32:32 2019
From: pypy.commits at gmail.com (arigo)
Date: Sun, 03 Nov 2019 06:32:32 -0800 (PST)
Subject: [pypy-commit] cffi release-1.13: Added tag v1.13.2 for changeset
 0c9be4c0d798
Message-ID: <5dbee500.1c69fb81.b6056.45c1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-1.13
Changeset: r3313:80521a7262a6
Date: 2019-11-03 15:29 +0100
http://bitbucket.org/cffi/cffi/changeset/80521a7262a6/

Log:	Added tag v1.13.2 for changeset 0c9be4c0d798

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -23,3 +23,4 @@
 bf80722dea36237710083315e336c81bc8571fd0 v1.12.3
 17c06e75f54cf5a31865961f0b498f99232a0bc8 v1.13.0
 9cc790d05bc5256eee32fddbe60aadf24f14ce44 v1.13.1
+0c9be4c0d79867d108c60ac3c66e2fb3e89aaae9 v1.13.2

From pypy.commits at gmail.com  Sun Nov  3 09:32:34 2019
From: pypy.commits at gmail.com (arigo)
Date: Sun, 03 Nov 2019 06:32:34 -0800 (PST)
Subject: [pypy-commit] cffi default: hg merge release-1.13
Message-ID: <5dbee502.1c69fb81.759a6.ab3b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3314:dca192bb54fd
Date: 2019-11-03 15:29 +0100
http://bitbucket.org/cffi/cffi/changeset/dca192bb54fd/

Log:	hg merge release-1.13

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -23,3 +23,4 @@
 bf80722dea36237710083315e336c81bc8571fd0 v1.12.3
 17c06e75f54cf5a31865961f0b498f99232a0bc8 v1.13.0
 9cc790d05bc5256eee32fddbe60aadf24f14ce44 v1.13.1
+0c9be4c0d79867d108c60ac3c66e2fb3e89aaae9 v1.13.2
diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -2,7 +2,7 @@
 #include <Python.h>
 #include "structmember.h"
 
-#define CFFI_VERSION  "1.13.1"
+#define CFFI_VERSION  "1.13.2"
 
 #ifdef MS_WIN32
 #include <windows.h>
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -14,7 +14,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.13.1", ("This test_c.py file is for testing a version"
+assert __version__ == "1.13.2", ("This test_c.py file is for testing a version"
                                  " of cffi that differs from the one that we"
                                  " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
diff --git a/cffi/__init__.py b/cffi/__init__.py
--- a/cffi/__init__.py
+++ b/cffi/__init__.py
@@ -5,8 +5,8 @@
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 from .error import PkgConfigError
 
-__version__ = "1.13.1"
-__version_info__ = (1, 13, 1)
+__version__ = "1.13.2"
+__version_info__ = (1, 13, 2)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/cffi/_embedding.h b/cffi/_embedding.h
--- a/cffi/_embedding.h
+++ b/cffi/_embedding.h
@@ -224,7 +224,7 @@
 
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.13.1"
+                               "\ncompiled with cffi version: 1.13.2"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/doc/source/conf.py b/doc/source/conf.py
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -47,7 +47,7 @@
 # The short X.Y version.
 version = '1.13'
 # The full version, including alpha/beta/rc tags.
-release = '1.13.1'
+release = '1.13.2'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -52,13 +52,13 @@
 
 * https://pypi.python.org/pypi/cffi
 
-* Checksums of the "source" package version 1.13.1:
+* Checksums of the "source" package version 1.13.2:
 
-   - MD5: 824ad9f228fbc6ce203c334e2ff4ab8f
+   - MD5: 652203cf99faa254efff7fab23c2f3a2
 
-   - SHA: 678dad3a3102ebcbeab8f5f132968afa444f6a54
+   - SHA: ad94c1b64570ec3016a945d4b226d527de81f9b9
 
-   - SHA256: 558b3afef987cf4b17abd849e7bedf64ee12b28175d564d05b628a0f9355599b
+   - SHA256: 599a1e8ff057ac530c9ad1778293c665cb81a791421f46922d80a86473c13346
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -2,6 +2,15 @@
 What's New
 ======================
 
+v1.13.2
+=======
+
+* re-release because the Linux wheels came with an attached version of libffi
+  that was very old and buggy (`issue #432`_).
+
+.. _`issue #432`: https://bitbucket.org/cffi/cffi/issues/432/
+
+
 
 v1.13.1
 =======
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -198,7 +198,7 @@
 
 `Mailing list <https://groups.google.com/forum/#!forum/python-cffi>`_
 """,
-        version='1.13.1',
+        version='1.13.2',
         packages=['cffi'] if cpython else [],
         package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h', 
                                '_embedding.h', '_cffi_errors.h']}

From pypy.commits at gmail.com  Sun Nov  3 15:06:06 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 03 Nov 2019 12:06:06 -0800 (PST)
Subject: [pypy-commit] buildbot default: use the libffi and openssl from
 pyca/cryptography to avoid bugs in centos6
Message-ID: <5dbf332e.1c69fb81.730c0.68ec@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r1100:ad3301acbb64
Date: 2019-11-03 15:05 -0500
http://bitbucket.org/pypy/buildbot/changeset/ad3301acbb64/

Log:	use the libffi and openssl from pyca/cryptography to avoid bugs in
	centos6

diff --git a/docker/Dockerfile b/docker/Dockerfile
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -2,7 +2,7 @@
 #  -t is the name of the image
 #  -f is this file
 #  the . is a random directory
-# docker build -t buildslave -f docker/Dockerfile .
+# docker build -t buildslave -f docker/Dockerfile docker 
 #
 # To create the buildslave configuration, call
 # docker run --rm --user $UID -v<abspath/to/builder/dir>:/build_dir> \
@@ -29,26 +29,39 @@
 #
 
 FROM centos:centos6
+WORKDIR /root
 
 RUN yum -y update
 RUN yum install -y wget bzip2-devel zlib-devel glibc-devel libX11-devel \
-   libXt-devel patch expat-devel libXft-devel openssl-devel tk-devel gdbm-devel \
-   perl xz-devel libffi-devel ncurses-devel sqlite-devel
-RUN yum install -y centos-release-scl
+   libXt-devel patch expat-devel libXft-devel tk-devel gdbm-devel \
+   perl xz-devel ncurses-devel sqlite-devel prelink
 
+# get the gcc-8 devtools and pypy
 RUN wget https://github.com/squeaky-pl/centos-devtools/releases/download/8.2-s1/gcc-8.2.0-binutils-2.32-x86_64.tar.bz2 -O - | tar -C / -xj
 RUN wget https://bitbucket.org/squeaky/portable-pypy/downloads/pypy-7.0.0-linux_x86_64-portable.tar.bz2 -O - | tar -C /opt -xj
 RUN ln -s /opt/pypy-7.0.0-linux_x86_64-portable/bin/pypy /usr/local/bin/pypy
 
+ENV PATH=/opt/devtools-8.2/bin:$PATH
+
+# Taken from pyca/infra/cryptography-manylinux
+# centos6 libffi is buggy, download and use a newer one
+# also use the version of openssl that latests pyca/cryptography uses
+ADD install_libffi.sh /root/install_libffi.sh
+ADD install_openssl.sh /root/install_openssl.sh
+RUN sh install_libffi.sh manylinux2010 2>&1 | tee /root/install_libffi.log
+RUN sh install_openssl.sh manylinux2010 2>&1 | tee /root/install_openssl.log
+
+RUN yum install -y centos-release-scl
 RUN yum install -y python27 python27-python-virtualenv
 
-#yuck
+# build a python 2.7 virtualenv, use the scl-installed version of python2.7
+# since centos6 itself has python2.6
 ENV LD_LIBRARY_PATH=/opt/rh/python27/root/usr/lib64
 RUN /opt/rh/python27/root/usr/bin/python -mvirtualenv /python27_virt
 ENV PATH=/python27_virt/bin:$PATH
-ENV PATH=/opt/devtools-8.2/bin:$PATH
 RUN pip install --upgrade pip setuptools
 RUN pip install buildbot-slave pytest hypothesis cffi vmprof mercurial
+ENV PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig
 
 CMD if [ -e /build_dir/buildbot.tac ]; then \
     buildslave start --nodaemon /build_dir; \
diff --git a/docker/install_libffi.sh b/docker/install_libffi.sh
new file mode 100755
--- /dev/null
+++ b/docker/install_libffi.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+set -xe
+
+LIBFFI_SHA256="d06ebb8e1d9a22d19e38d63fdb83954253f39bedc5d46232a05645685722ca37"
+LIBFFI_VERSION="3.2.1"
+
+function check_sha256sum {
+    local fname=$1
+    local sha256=$2
+    echo "${sha256}  ${fname}" > "${fname}.sha256"
+    sha256sum -c "${fname}.sha256"
+    rm "${fname}.sha256"
+}
+
+curl -#O "https://mirrors.ocf.berkeley.edu/debian/pool/main/libf/libffi/libffi_${LIBFFI_VERSION}.orig.tar.gz"
+check_sha256sum "libffi_${LIBFFI_VERSION}.orig.tar.gz" ${LIBFFI_SHA256}
+tar zxf libffi*.orig.tar.gz
+PATH=/opt/perl/bin:$PATH
+pushd libffi*
+if [ "$1" == "manylinux1" ]; then
+  STACK_PROTECTOR_FLAGS="-fstack-protector --param=ssp-buffer-size=4"
+else
+  STACK_PROTECTOR_FLAGS="-fstack-protector-strong"
+fi
+./configure --prefix=/usr/local CFLAGS="-g -O2 $STACK_PROTECTOR_FLAGS -Wformat -Werror=format-security"
+make install
+popd
+rm -rf libffi*
diff --git a/docker/install_openssl.sh b/docker/install_openssl.sh
new file mode 100755
--- /dev/null
+++ b/docker/install_openssl.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+set -xe
+
+OPENSSL_URL="https://www.openssl.org/source/"
+OPENSSL_NAME="openssl-1.1.1d"
+OPENSSL_SHA256="1e3a91bc1f9dfce01af26026f856e064eab4c8ee0a8f457b5ae30b40b8b711f2"
+
+function check_sha256sum {
+    local fname=$1
+    local sha256=$2
+    echo "${sha256}  ${fname}" > "${fname}.sha256"
+    sha256sum -c "${fname}.sha256"
+    rm "${fname}.sha256"
+}
+
+curl -#O "${OPENSSL_URL}/${OPENSSL_NAME}.tar.gz"
+check_sha256sum ${OPENSSL_NAME}.tar.gz ${OPENSSL_SHA256}
+tar zxf ${OPENSSL_NAME}.tar.gz
+PATH=/opt/perl/bin:$PATH
+pushd ${OPENSSL_NAME}
+./config no-comp enable-ec_nistp_64_gcc_128 no-shared no-dynamic-engine --prefix=/usr/local --openssldir=/usr/local
+make depend
+make -j4
+# avoid installing the docs
+# https://github.com/openssl/openssl/issues/6685#issuecomment-403838728
+make install_sw install_ssldirs
+popd
+rm -rf openssl*

From pypy.commits at gmail.com  Mon Nov  4 05:33:41 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 04 Nov 2019 02:33:41 -0800 (PST)
Subject: [pypy-commit] pypy expose-jsonmap: expose dictstructure.append,
 which needs a bit of refactoring
Message-ID: <5dbffe85.1c69fb81.4e2ad.3a1e@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: expose-jsonmap
Changeset: r97947:68a281f37aaa
Date: 2019-11-04 11:22 +0100
http://bitbucket.org/pypy/pypy/changeset/68a281f37aaa/

Log:	expose dictstructure.append, which needs a bit of refactoring

diff --git a/pypy/module/__pypy__/interp_dictstructure.py b/pypy/module/__pypy__/interp_dictstructure.py
--- a/pypy/module/__pypy__/interp_dictstructure.py
+++ b/pypy/module/__pypy__/interp_dictstructure.py
@@ -16,7 +16,7 @@
         if type(w_x) is not W_UnicodeObject:
             raise oefmt(space.w_TypeError, "expected unicode, got %T", w_x)
         u = space.utf8_w(w_x)
-        m = m.get_next(w_x, u, 0, len(u), terminator)
+        m = m.get_next(w_x, u, 0, len(u))
         if m is None:
             raise oefmt(space.w_ValueError, "repeated key %R", w_x)
         if not m.is_state_useful():
diff --git a/pypy/module/__pypy__/test/test_dictstructure.py b/pypy/module/__pypy__/test/test_dictstructure.py
--- a/pypy/module/__pypy__/test/test_dictstructure.py
+++ b/pypy/module/__pypy__/test/test_dictstructure.py
@@ -29,3 +29,13 @@
         from __pypy__ import newdictstructure
         with raises(ValueError):
             newdictstructure([u"a", u"a"])
+
+    def test_getnext(self):
+        from __pypy__ import newdictstructure
+        s = newdictstructure([u"a"])
+        s1 = s.append(u"b")
+        assert s.append(u"b") is s1
+        assert s1.instantiate_dict([1, 2]) == {u"a": 1, u"b": 2}
+        with raises(ValueError):
+            s.append(u"a")
+
diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -638,7 +638,7 @@
             self._raise("Key name must be string at char %d", i)
         i += 1
         w_key = self._decode_key_string(i)
-        return currmap.get_next(w_key, self.s, start, self.pos, self.startmap)
+        return currmap.get_next(w_key, self.s, start, self.pos)
 
     def _decode_key_string(self, i):
         """ decode key at position i as a string. Key strings are always
@@ -800,7 +800,7 @@
         elif self.nextmap_first:
             self.nextmap_first._check_invariants()
 
-    def get_next(self, w_key, string, start, stop, terminator):
+    def get_next(self, w_key, string, start, stop):
         """ Returns the next map, given a wrapped key w_key, the json input
         string with positions start and stop, as well as a terminator.
 
@@ -819,7 +819,7 @@
 
         if nextmap_first is None:
             # first transition ever seen, don't initialize nextmap_all
-            next = self._make_next_map(w_key, string[start:stop])
+            next, terminator = self._make_next_map(w_key, string[start:stop])
             if next is None:
                 return None
             self.nextmap_first = next
@@ -835,7 +835,7 @@
                     return next
             # if we are at this point we didn't find the transition yet, so
             # create a new one
-            next = self._make_next_map(w_key, string[start:stop])
+            next, terminator = self._make_next_map(w_key, string[start:stop])
             if next is None:
                 return None
             self.nextmap_all[w_key] = next
@@ -886,9 +886,10 @@
         check = self
         while isinstance(check, JSONMap):
             if check.w_key._utf8 == w_key._utf8:
-                return None
+                return None, None
             check = check.prev
-        return JSONMap(self.space, self, w_key, key_repr)
+        assert isinstance(check, Terminator)
+        return JSONMap(self.space, self, w_key, key_repr), check
 
     def fill_dict(self, dict_w, values_w):
         """ recursively fill the dictionary dict_w in the correct order,
@@ -899,21 +900,36 @@
     # exposed methods
 
     def descr_instantiate_dict(self, space, w_l):
-        from pypy.objspace.std.jsondict import from_values_and_jsonmap
+        """ Create a dict instance given from the structure, using the list l
+        as values. It must have the same length as the structure has keys. """
+        from pypy.objspace.std.jsondict import from_values_and_jsonmap_checked
         from pypy.objspace.std.jsondict import devolve_jsonmap_dict
         l_w = space.listview(w_l)
         if not isinstance(self, JSONMap):
             return space.newdict()
-        keys_w = self.get_keys_in_order()
-        if len(l_w) != len(keys_w):
-            raise oefmt(space.w_ValueError, "expected %s values, got %s", str(len(keys_w)), str(len(l_w)))
-        w_dict = from_values_and_jsonmap(self.space, l_w[:], self)
+        w_dict = from_values_and_jsonmap_checked(self.space, l_w[:], self)
         if self.is_state_blocked():
             devolve_jsonmap_dict(w_dict)
         return w_dict
 
     def descr_repr(self, space):
-        return space.newtext("<DictStructure [%s]>" % ", ".join([space.text_w(space.repr(w_key)) for w_key in self.get_keys_in_order()]))
+        res = []
+        curr = self
+        while type(curr) is JSONMap:
+            res.append(space.text_w(space.repr(self.w_key)))
+            curr = curr.prev
+        res.reverse()
+        return space.newtext("<DictStructure [%s]>" % ", ".join(res))
+
+    def descr_append(self, space, w_u):
+        from pypy.objspace.std.unicodeobject import W_UnicodeObject
+        if type(w_u) is not W_UnicodeObject:
+            raise oefmt(space.w_TypeError, "expected unicode, got %T", w_u)
+        u = space.utf8_w(w_u)
+        res = self.get_next(w_u, u, 0, len(u))
+        if res is None:
+            raise oefmt(space.w_ValueError, "repeated key %R", w_u)
+        return res
 
 
     # ____________________________________________________________
@@ -1190,5 +1206,6 @@
     __repr__ = interp2app(MapBase.descr_repr),
     last_key = GetSetProperty(get_last_key, name="last_key"),
     previous = GetSetProperty(get_previous, name="previous"),
+    append = interp2app(MapBase.descr_append),
 )
 MapBase.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py
--- a/pypy/module/_pypyjson/test/test__pypyjson.py
+++ b/pypy/module/_pypyjson/test/test__pypyjson.py
@@ -19,7 +19,7 @@
         w_a = self.space.newutf8("a", 1)
         w_b = self.space.newutf8("b", 1)
         w_c = self.space.newutf8("c", 1)
-        m1 = m.get_next(w_a, '"a"', 0, 3, m)
+        m1 = m.get_next(w_a, '"a"', 0, 3)
         assert m1.w_key == w_a
         assert m1.nextmap_first is None
         assert m1.key_repr == '"a"'
@@ -27,16 +27,16 @@
         assert not m1.key_repr_cmp('b": 123', 0)
         assert m.nextmap_first.w_key == w_a
 
-        m2 = m.get_next(w_a, '"a"', 0, 3, m)
+        m2 = m.get_next(w_a, '"a"', 0, 3)
         assert m2 is m1
 
-        m3 = m.get_next(w_b, '"b"', 0, 3, m)
+        m3 = m.get_next(w_b, '"b"', 0, 3)
         assert m3.w_key == w_b
         assert m3.nextmap_first is None
         assert m3.key_repr == '"b"'
         assert m.nextmap_first is m1
 
-        m4 = m3.get_next(w_c, '"c"', 0, 3, m)
+        m4 = m3.get_next(w_c, '"c"', 0, 3)
         assert m4.w_key == w_c
         assert m4.nextmap_first is None
         assert m4.key_repr == '"c"'
@@ -49,9 +49,9 @@
         w_a = space.newutf8("a", 1)
         w_b = space.newutf8("b", 1)
         w_c = space.newutf8("c", 1)
-        m1 = m.get_next(w_a, 'a"', 0, 2, m)
-        m2 = m1.get_next(w_b, 'b"', 0, 2, m)
-        m3 = m2.get_next(w_c, 'c"', 0, 2, m)
+        m1 = m.get_next(w_a, 'a"', 0, 2)
+        m2 = m1.get_next(w_b, 'b"', 0, 2)
+        m3 = m2.get_next(w_c, 'c"', 0, 2)
         d = OrderedDict()
         m3.fill_dict(d, [space.w_None, space.w_None, space.w_None])
         assert list(d) == [w_a, w_b, w_c]
@@ -61,10 +61,10 @@
         w_a = self.space.newutf8("a", 1)
         w_b = self.space.newutf8("b", 1)
         w_c = self.space.newutf8("c", 1)
-        m1 = m.get_next(w_a, '"a"', 0, 3, m)
-        m1 = m1.get_next(w_b, '"b"', 0, 3, m)
-        m1 = m1.get_next(w_c, '"c"', 0, 3, m)
-        m2 = m1.get_next(w_a, '"a"', 0, 3, m)
+        m1 = m.get_next(w_a, '"a"', 0, 3)
+        m1 = m1.get_next(w_b, '"b"', 0, 3)
+        m1 = m1.get_next(w_c, '"c"', 0, 3)
+        m2 = m1.get_next(w_a, '"a"', 0, 3)
         assert m2 is None
 
 
@@ -111,10 +111,10 @@
         w_d = self.space.newutf8("d", 1)
         base = Terminator(self.space)
         base.instantiation_count = 6
-        m1 = base.get_next(w_a, 'a"', 0, 2, base)
-        m2 = m1.get_next(w_b, 'b"', 0, 2, base)
-        m3 = m2.get_next(w_c, 'c"', 0, 2, base)
-        m4 = m2.get_next(w_d, 'd"', 0, 2, base)
+        m1 = base.get_next(w_a, 'a"', 0, 2)
+        m2 = m1.get_next(w_b, 'b"', 0, 2)
+        m3 = m2.get_next(w_c, 'c"', 0, 2)
+        m4 = m2.get_next(w_d, 'd"', 0, 2)
         return base, m1, m2, m3, m4
 
     # unit tests for map state transistions
@@ -152,7 +152,7 @@
         assert m2.number_of_leaves == 2
         assert m3.number_of_leaves == 1
         assert m4.number_of_leaves == 1
-        m5 = m2.get_next(w_x, 'x"', 0, 2, base)
+        m5 = m2.get_next(w_x, 'x"', 0, 2)
         assert base.number_of_leaves == 3
         assert m1.number_of_leaves == 3
         assert m2.number_of_leaves == 3
@@ -161,7 +161,7 @@
     def test_number_of_leaves_after_mark_blocked(self):
         w_x = self.space.newutf8("x", 1)
         base, m1, m2, m3, m4 = self._make_some_maps()
-        m5 = m2.get_next(w_x, 'x"', 0, 2, base)
+        m5 = m2.get_next(w_x, 'x"', 0, 2)
         assert base.number_of_leaves == 3
         m2.mark_blocked(base)
         assert base.number_of_leaves == 1
@@ -186,11 +186,11 @@
         w_d = self.space.newutf8("d", 1)
         base = Terminator(self.space)
         base.instantiation_count = 6
-        m1 = base.get_next(w_a, 'a"', 0, 2, base)
-        m2 = base.get_next(w_b, 'b"', 0, 2, base)
-        m3 = base.get_next(w_c, 'c"', 0, 2, base)
-        m4 = base.get_next(w_d, 'd"', 0, 2, base)
-        m5 = m4.get_next(w_a, 'a"', 0, 2, base)
+        m1 = base.get_next(w_a, 'a"', 0, 2)
+        m2 = base.get_next(w_b, 'b"', 0, 2)
+        m3 = base.get_next(w_c, 'c"', 0, 2)
+        m4 = base.get_next(w_d, 'd"', 0, 2)
+        m5 = m4.get_next(w_a, 'a"', 0, 2)
         base.instantiation_count = 7
         m1.instantiation_count = 2
         m2.instantiation_count = 2
@@ -216,8 +216,8 @@
         s = '{"a": 1, "b": 2, "c": 3}'
         dec = JSONDecoder(space, s)
         dec.startmap = base = Terminator(space)
-        m1 = base.get_next(w_a, 'a"', 0, 2, base)
-        m2 = m1.get_next(w_b, 'b"', 0, 2, base)
+        m1 = base.get_next(w_a, 'a"', 0, 2)
+        m2 = m1.get_next(w_b, 'b"', 0, 2)
         m2.mark_blocked(base)
         w_res = dec.decode_object(1)
         assert space.int_w(space.len(w_res)) == 3
@@ -233,10 +233,10 @@
         w_u = self.space.newutf8("u", 1)
         space = self.space
         base = Terminator(space)
-        m1 = base.get_next(w_a, 'a"', 0, 2, base)
-        m2 = m1.get_next(w_b, 'b"', 0, 2, base)
-        m2.get_next(w_x, 'x"', 0, 2, base)
-        m2.get_next(w_u, 'u"', 0, 2, base)
+        m1 = base.get_next(w_a, 'a"', 0, 2)
+        m2 = m1.get_next(w_b, 'b"', 0, 2)
+        m2.get_next(w_x, 'x"', 0, 2)
+        m2.get_next(w_u, 'u"', 0, 2)
         assert base.number_of_leaves == 2
         m2.mark_blocked(base)
         assert base.number_of_leaves == 1
diff --git a/pypy/objspace/std/jsondict.py b/pypy/objspace/std/jsondict.py
--- a/pypy/objspace/std/jsondict.py
+++ b/pypy/objspace/std/jsondict.py
@@ -8,7 +8,7 @@
 from pypy.objspace.std.dictmultiobject import (
     UnicodeDictStrategy, DictStrategy,
     create_iterator_classes, W_DictObject)
-
+from pypy.interpreter.error import oefmt
 
 def from_values_and_jsonmap(space, values_w, jsonmap):
     debug.make_sure_not_resized(values_w)
@@ -21,6 +21,18 @@
     storage = strategy.erase(values_w)
     return W_DictObject(space, strategy, storage)
 
+def from_values_and_jsonmap_checked(space, values_w, jsonmap):
+    debug.make_sure_not_resized(values_w)
+    strategy = jsonmap.strategy_instance
+    if strategy is None:
+        jsonmap.strategy_instance = strategy = JsonDictStrategy(space, jsonmap)
+    keys_w = strategy.get_keys_in_order()
+    if len(values_w) != len(keys_w):
+        raise oefmt(space.w_ValueError, "expected %s values, got %s", str(len(keys_w)), str(len(values_w)))
+    storage = strategy.erase(values_w)
+    return W_DictObject(space, strategy, storage)
+
+
 def devolve_jsonmap_dict(w_dict):
     assert isinstance(w_dict, W_DictObject)
     strategy = w_dict.get_strategy()
diff --git a/pypy/objspace/std/test/test_jsondict.py b/pypy/objspace/std/test/test_jsondict.py
--- a/pypy/objspace/std/test/test_jsondict.py
+++ b/pypy/objspace/std/test/test_jsondict.py
@@ -10,25 +10,23 @@
         w_a = space.newutf8("a", 1)
         w_b = space.newutf8("b", 1)
         w_c = space.newutf8("c", 1)
-        m1 = m.get_next(w_a, 'a"', 0, 2, m)
+        m1 = m.get_next(w_a, 'a"', 0, 2)
         dm1 = JsonDictStrategy(space, m1)
         assert dm1.get_index(w_a) == 0
         assert dm1.get_index(w_b) == -1
 
-        m2 = m.get_next(w_b, 'b"', 0, 2, m)
+        m2 = m.get_next(w_b, 'b"', 0, 2)
         dm2 = JsonDictStrategy(space, m2)
         assert dm2.get_index(w_b) == 0
         assert dm2.get_index(w_a) == -1
 
-        m3 = m2.get_next(w_c, 'c"', 0, 2, m)
+        m3 = m2.get_next(w_c, 'c"', 0, 2)
         dm3 = JsonDictStrategy(space, m3)
         assert dm3.get_index(w_b) == 0
         assert dm3.get_index(w_c) == 1
         assert dm3.get_index(w_a) == -1
 
 
-
-
 class AppTest(object):
     spaceconfig = {"objspace.usemodules._pypyjson": True}
 
@@ -150,8 +148,8 @@
         w_a = self.space.newutf8("a", 1)
         w_b = self.space.newutf8("b", 1)
         base = interp_decoder.Terminator(space)
-        m1 = base.get_next(w_a, 'a"', 0, 2, base)
-        m2 = m1.get_next(w_b, 'b"', 0, 2, base)
+        m1 = base.get_next(w_a, 'a"', 0, 2)
+        m2 = m1.get_next(w_b, 'b"', 0, 2)
 
         w_d = from_values_and_jsonmap(space, [w_a, w_b], m2)
         return base, m2, w_d, w_a, w_b

From pypy.commits at gmail.com  Mon Nov  4 05:53:55 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 04 Nov 2019 02:53:55 -0800 (PST)
Subject: [pypy-commit] pypy expose-jsonmap: Backed out changeset
 68a281f37aaa, going for different approach
Message-ID: <5dc00343.1c69fb81.4a6e2.558c@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: expose-jsonmap
Changeset: r97948:51ec4a0ae27e
Date: 2019-11-04 11:45 +0100
http://bitbucket.org/pypy/pypy/changeset/51ec4a0ae27e/

Log:	Backed out changeset 68a281f37aaa, going for different approach

diff --git a/pypy/module/__pypy__/interp_dictstructure.py b/pypy/module/__pypy__/interp_dictstructure.py
--- a/pypy/module/__pypy__/interp_dictstructure.py
+++ b/pypy/module/__pypy__/interp_dictstructure.py
@@ -16,7 +16,7 @@
         if type(w_x) is not W_UnicodeObject:
             raise oefmt(space.w_TypeError, "expected unicode, got %T", w_x)
         u = space.utf8_w(w_x)
-        m = m.get_next(w_x, u, 0, len(u))
+        m = m.get_next(w_x, u, 0, len(u), terminator)
         if m is None:
             raise oefmt(space.w_ValueError, "repeated key %R", w_x)
         if not m.is_state_useful():
diff --git a/pypy/module/__pypy__/test/test_dictstructure.py b/pypy/module/__pypy__/test/test_dictstructure.py
--- a/pypy/module/__pypy__/test/test_dictstructure.py
+++ b/pypy/module/__pypy__/test/test_dictstructure.py
@@ -29,13 +29,3 @@
         from __pypy__ import newdictstructure
         with raises(ValueError):
             newdictstructure([u"a", u"a"])
-
-    def test_getnext(self):
-        from __pypy__ import newdictstructure
-        s = newdictstructure([u"a"])
-        s1 = s.append(u"b")
-        assert s.append(u"b") is s1
-        assert s1.instantiate_dict([1, 2]) == {u"a": 1, u"b": 2}
-        with raises(ValueError):
-            s.append(u"a")
-
diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -638,7 +638,7 @@
             self._raise("Key name must be string at char %d", i)
         i += 1
         w_key = self._decode_key_string(i)
-        return currmap.get_next(w_key, self.s, start, self.pos)
+        return currmap.get_next(w_key, self.s, start, self.pos, self.startmap)
 
     def _decode_key_string(self, i):
         """ decode key at position i as a string. Key strings are always
@@ -800,7 +800,7 @@
         elif self.nextmap_first:
             self.nextmap_first._check_invariants()
 
-    def get_next(self, w_key, string, start, stop):
+    def get_next(self, w_key, string, start, stop, terminator):
         """ Returns the next map, given a wrapped key w_key, the json input
         string with positions start and stop, as well as a terminator.
 
@@ -819,7 +819,7 @@
 
         if nextmap_first is None:
             # first transition ever seen, don't initialize nextmap_all
-            next, terminator = self._make_next_map(w_key, string[start:stop])
+            next = self._make_next_map(w_key, string[start:stop])
             if next is None:
                 return None
             self.nextmap_first = next
@@ -835,7 +835,7 @@
                     return next
             # if we are at this point we didn't find the transition yet, so
             # create a new one
-            next, terminator = self._make_next_map(w_key, string[start:stop])
+            next = self._make_next_map(w_key, string[start:stop])
             if next is None:
                 return None
             self.nextmap_all[w_key] = next
@@ -886,10 +886,9 @@
         check = self
         while isinstance(check, JSONMap):
             if check.w_key._utf8 == w_key._utf8:
-                return None, None
+                return None
             check = check.prev
-        assert isinstance(check, Terminator)
-        return JSONMap(self.space, self, w_key, key_repr), check
+        return JSONMap(self.space, self, w_key, key_repr)
 
     def fill_dict(self, dict_w, values_w):
         """ recursively fill the dictionary dict_w in the correct order,
@@ -900,36 +899,21 @@
     # exposed methods
 
     def descr_instantiate_dict(self, space, w_l):
-        """ Create a dict instance given from the structure, using the list l
-        as values. It must have the same length as the structure has keys. """
-        from pypy.objspace.std.jsondict import from_values_and_jsonmap_checked
+        from pypy.objspace.std.jsondict import from_values_and_jsonmap
         from pypy.objspace.std.jsondict import devolve_jsonmap_dict
         l_w = space.listview(w_l)
         if not isinstance(self, JSONMap):
             return space.newdict()
-        w_dict = from_values_and_jsonmap_checked(self.space, l_w[:], self)
+        keys_w = self.get_keys_in_order()
+        if len(l_w) != len(keys_w):
+            raise oefmt(space.w_ValueError, "expected %s values, got %s", str(len(keys_w)), str(len(l_w)))
+        w_dict = from_values_and_jsonmap(self.space, l_w[:], self)
         if self.is_state_blocked():
             devolve_jsonmap_dict(w_dict)
         return w_dict
 
     def descr_repr(self, space):
-        res = []
-        curr = self
-        while type(curr) is JSONMap:
-            res.append(space.text_w(space.repr(self.w_key)))
-            curr = curr.prev
-        res.reverse()
-        return space.newtext("<DictStructure [%s]>" % ", ".join(res))
-
-    def descr_append(self, space, w_u):
-        from pypy.objspace.std.unicodeobject import W_UnicodeObject
-        if type(w_u) is not W_UnicodeObject:
-            raise oefmt(space.w_TypeError, "expected unicode, got %T", w_u)
-        u = space.utf8_w(w_u)
-        res = self.get_next(w_u, u, 0, len(u))
-        if res is None:
-            raise oefmt(space.w_ValueError, "repeated key %R", w_u)
-        return res
+        return space.newtext("<DictStructure [%s]>" % ", ".join([space.text_w(space.repr(w_key)) for w_key in self.get_keys_in_order()]))
 
 
     # ____________________________________________________________
@@ -1206,6 +1190,5 @@
     __repr__ = interp2app(MapBase.descr_repr),
     last_key = GetSetProperty(get_last_key, name="last_key"),
     previous = GetSetProperty(get_previous, name="previous"),
-    append = interp2app(MapBase.descr_append),
 )
 MapBase.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py
--- a/pypy/module/_pypyjson/test/test__pypyjson.py
+++ b/pypy/module/_pypyjson/test/test__pypyjson.py
@@ -19,7 +19,7 @@
         w_a = self.space.newutf8("a", 1)
         w_b = self.space.newutf8("b", 1)
         w_c = self.space.newutf8("c", 1)
-        m1 = m.get_next(w_a, '"a"', 0, 3)
+        m1 = m.get_next(w_a, '"a"', 0, 3, m)
         assert m1.w_key == w_a
         assert m1.nextmap_first is None
         assert m1.key_repr == '"a"'
@@ -27,16 +27,16 @@
         assert not m1.key_repr_cmp('b": 123', 0)
         assert m.nextmap_first.w_key == w_a
 
-        m2 = m.get_next(w_a, '"a"', 0, 3)
+        m2 = m.get_next(w_a, '"a"', 0, 3, m)
         assert m2 is m1
 
-        m3 = m.get_next(w_b, '"b"', 0, 3)
+        m3 = m.get_next(w_b, '"b"', 0, 3, m)
         assert m3.w_key == w_b
         assert m3.nextmap_first is None
         assert m3.key_repr == '"b"'
         assert m.nextmap_first is m1
 
-        m4 = m3.get_next(w_c, '"c"', 0, 3)
+        m4 = m3.get_next(w_c, '"c"', 0, 3, m)
         assert m4.w_key == w_c
         assert m4.nextmap_first is None
         assert m4.key_repr == '"c"'
@@ -49,9 +49,9 @@
         w_a = space.newutf8("a", 1)
         w_b = space.newutf8("b", 1)
         w_c = space.newutf8("c", 1)
-        m1 = m.get_next(w_a, 'a"', 0, 2)
-        m2 = m1.get_next(w_b, 'b"', 0, 2)
-        m3 = m2.get_next(w_c, 'c"', 0, 2)
+        m1 = m.get_next(w_a, 'a"', 0, 2, m)
+        m2 = m1.get_next(w_b, 'b"', 0, 2, m)
+        m3 = m2.get_next(w_c, 'c"', 0, 2, m)
         d = OrderedDict()
         m3.fill_dict(d, [space.w_None, space.w_None, space.w_None])
         assert list(d) == [w_a, w_b, w_c]
@@ -61,10 +61,10 @@
         w_a = self.space.newutf8("a", 1)
         w_b = self.space.newutf8("b", 1)
         w_c = self.space.newutf8("c", 1)
-        m1 = m.get_next(w_a, '"a"', 0, 3)
-        m1 = m1.get_next(w_b, '"b"', 0, 3)
-        m1 = m1.get_next(w_c, '"c"', 0, 3)
-        m2 = m1.get_next(w_a, '"a"', 0, 3)
+        m1 = m.get_next(w_a, '"a"', 0, 3, m)
+        m1 = m1.get_next(w_b, '"b"', 0, 3, m)
+        m1 = m1.get_next(w_c, '"c"', 0, 3, m)
+        m2 = m1.get_next(w_a, '"a"', 0, 3, m)
         assert m2 is None
 
 
@@ -111,10 +111,10 @@
         w_d = self.space.newutf8("d", 1)
         base = Terminator(self.space)
         base.instantiation_count = 6
-        m1 = base.get_next(w_a, 'a"', 0, 2)
-        m2 = m1.get_next(w_b, 'b"', 0, 2)
-        m3 = m2.get_next(w_c, 'c"', 0, 2)
-        m4 = m2.get_next(w_d, 'd"', 0, 2)
+        m1 = base.get_next(w_a, 'a"', 0, 2, base)
+        m2 = m1.get_next(w_b, 'b"', 0, 2, base)
+        m3 = m2.get_next(w_c, 'c"', 0, 2, base)
+        m4 = m2.get_next(w_d, 'd"', 0, 2, base)
         return base, m1, m2, m3, m4
 
     # unit tests for map state transistions
@@ -152,7 +152,7 @@
         assert m2.number_of_leaves == 2
         assert m3.number_of_leaves == 1
         assert m4.number_of_leaves == 1
-        m5 = m2.get_next(w_x, 'x"', 0, 2)
+        m5 = m2.get_next(w_x, 'x"', 0, 2, base)
         assert base.number_of_leaves == 3
         assert m1.number_of_leaves == 3
         assert m2.number_of_leaves == 3
@@ -161,7 +161,7 @@
     def test_number_of_leaves_after_mark_blocked(self):
         w_x = self.space.newutf8("x", 1)
         base, m1, m2, m3, m4 = self._make_some_maps()
-        m5 = m2.get_next(w_x, 'x"', 0, 2)
+        m5 = m2.get_next(w_x, 'x"', 0, 2, base)
         assert base.number_of_leaves == 3
         m2.mark_blocked(base)
         assert base.number_of_leaves == 1
@@ -186,11 +186,11 @@
         w_d = self.space.newutf8("d", 1)
         base = Terminator(self.space)
         base.instantiation_count = 6
-        m1 = base.get_next(w_a, 'a"', 0, 2)
-        m2 = base.get_next(w_b, 'b"', 0, 2)
-        m3 = base.get_next(w_c, 'c"', 0, 2)
-        m4 = base.get_next(w_d, 'd"', 0, 2)
-        m5 = m4.get_next(w_a, 'a"', 0, 2)
+        m1 = base.get_next(w_a, 'a"', 0, 2, base)
+        m2 = base.get_next(w_b, 'b"', 0, 2, base)
+        m3 = base.get_next(w_c, 'c"', 0, 2, base)
+        m4 = base.get_next(w_d, 'd"', 0, 2, base)
+        m5 = m4.get_next(w_a, 'a"', 0, 2, base)
         base.instantiation_count = 7
         m1.instantiation_count = 2
         m2.instantiation_count = 2
@@ -216,8 +216,8 @@
         s = '{"a": 1, "b": 2, "c": 3}'
         dec = JSONDecoder(space, s)
         dec.startmap = base = Terminator(space)
-        m1 = base.get_next(w_a, 'a"', 0, 2)
-        m2 = m1.get_next(w_b, 'b"', 0, 2)
+        m1 = base.get_next(w_a, 'a"', 0, 2, base)
+        m2 = m1.get_next(w_b, 'b"', 0, 2, base)
         m2.mark_blocked(base)
         w_res = dec.decode_object(1)
         assert space.int_w(space.len(w_res)) == 3
@@ -233,10 +233,10 @@
         w_u = self.space.newutf8("u", 1)
         space = self.space
         base = Terminator(space)
-        m1 = base.get_next(w_a, 'a"', 0, 2)
-        m2 = m1.get_next(w_b, 'b"', 0, 2)
-        m2.get_next(w_x, 'x"', 0, 2)
-        m2.get_next(w_u, 'u"', 0, 2)
+        m1 = base.get_next(w_a, 'a"', 0, 2, base)
+        m2 = m1.get_next(w_b, 'b"', 0, 2, base)
+        m2.get_next(w_x, 'x"', 0, 2, base)
+        m2.get_next(w_u, 'u"', 0, 2, base)
         assert base.number_of_leaves == 2
         m2.mark_blocked(base)
         assert base.number_of_leaves == 1
diff --git a/pypy/objspace/std/jsondict.py b/pypy/objspace/std/jsondict.py
--- a/pypy/objspace/std/jsondict.py
+++ b/pypy/objspace/std/jsondict.py
@@ -8,7 +8,7 @@
 from pypy.objspace.std.dictmultiobject import (
     UnicodeDictStrategy, DictStrategy,
     create_iterator_classes, W_DictObject)
-from pypy.interpreter.error import oefmt
+
 
 def from_values_and_jsonmap(space, values_w, jsonmap):
     debug.make_sure_not_resized(values_w)
@@ -21,18 +21,6 @@
     storage = strategy.erase(values_w)
     return W_DictObject(space, strategy, storage)
 
-def from_values_and_jsonmap_checked(space, values_w, jsonmap):
-    debug.make_sure_not_resized(values_w)
-    strategy = jsonmap.strategy_instance
-    if strategy is None:
-        jsonmap.strategy_instance = strategy = JsonDictStrategy(space, jsonmap)
-    keys_w = strategy.get_keys_in_order()
-    if len(values_w) != len(keys_w):
-        raise oefmt(space.w_ValueError, "expected %s values, got %s", str(len(keys_w)), str(len(values_w)))
-    storage = strategy.erase(values_w)
-    return W_DictObject(space, strategy, storage)
-
-
 def devolve_jsonmap_dict(w_dict):
     assert isinstance(w_dict, W_DictObject)
     strategy = w_dict.get_strategy()
diff --git a/pypy/objspace/std/test/test_jsondict.py b/pypy/objspace/std/test/test_jsondict.py
--- a/pypy/objspace/std/test/test_jsondict.py
+++ b/pypy/objspace/std/test/test_jsondict.py
@@ -10,23 +10,25 @@
         w_a = space.newutf8("a", 1)
         w_b = space.newutf8("b", 1)
         w_c = space.newutf8("c", 1)
-        m1 = m.get_next(w_a, 'a"', 0, 2)
+        m1 = m.get_next(w_a, 'a"', 0, 2, m)
         dm1 = JsonDictStrategy(space, m1)
         assert dm1.get_index(w_a) == 0
         assert dm1.get_index(w_b) == -1
 
-        m2 = m.get_next(w_b, 'b"', 0, 2)
+        m2 = m.get_next(w_b, 'b"', 0, 2, m)
         dm2 = JsonDictStrategy(space, m2)
         assert dm2.get_index(w_b) == 0
         assert dm2.get_index(w_a) == -1
 
-        m3 = m2.get_next(w_c, 'c"', 0, 2)
+        m3 = m2.get_next(w_c, 'c"', 0, 2, m)
         dm3 = JsonDictStrategy(space, m3)
         assert dm3.get_index(w_b) == 0
         assert dm3.get_index(w_c) == 1
         assert dm3.get_index(w_a) == -1
 
 
+
+
 class AppTest(object):
     spaceconfig = {"objspace.usemodules._pypyjson": True}
 
@@ -148,8 +150,8 @@
         w_a = self.space.newutf8("a", 1)
         w_b = self.space.newutf8("b", 1)
         base = interp_decoder.Terminator(space)
-        m1 = base.get_next(w_a, 'a"', 0, 2)
-        m2 = m1.get_next(w_b, 'b"', 0, 2)
+        m1 = base.get_next(w_a, 'a"', 0, 2, base)
+        m2 = m1.get_next(w_b, 'b"', 0, 2, base)
 
         w_d = from_values_and_jsonmap(space, [w_a, w_b], m2)
         return base, m2, w_d, w_a, w_b

From pypy.commits at gmail.com  Mon Nov  4 05:53:56 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 04 Nov 2019 02:53:56 -0800 (PST)
Subject: [pypy-commit] pypy expose-jsonmap: re-implement
 dictstructure.append, with proper statistics keeping
Message-ID: <5dc00344.1c69fb81.bf01f.f4e6@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: expose-jsonmap
Changeset: r97949:c4a8e1c5dfd0
Date: 2019-11-04 11:53 +0100
http://bitbucket.org/pypy/pypy/changeset/c4a8e1c5dfd0/

Log:	re-implement dictstructure.append, with proper statistics keeping

diff --git a/pypy/module/__pypy__/test/test_dictstructure.py b/pypy/module/__pypy__/test/test_dictstructure.py
--- a/pypy/module/__pypy__/test/test_dictstructure.py
+++ b/pypy/module/__pypy__/test/test_dictstructure.py
@@ -29,3 +29,25 @@
         from __pypy__ import newdictstructure
         with raises(ValueError):
             newdictstructure([u"a", u"a"])
+
+    def test_append(self):
+        from __pypy__ import newdictstructure
+        s = newdictstructure([u"a"])
+        s1 = s.append(u"b")
+        assert s.append(u"b") is s1
+        assert s1.instantiate_dict([1, 2]) == {u"a": 1, u"b": 2}
+        with raises(ValueError):
+            s.append(u"a")
+
+
+class TestDictStructure(object):
+    def test_append_and_transitions(self):
+        from pypy.module._pypyjson.interp_decoder import Terminator
+        m = Terminator(self.space)
+        w_a = self.space.newutf8("a", 1)
+        w_b = self.space.newutf8("b", 1)
+        m1 = m.descr_append(self.space, w_a)
+        m2 = m1.descr_append(self.space, w_b)
+        count = m2.instantiation_count
+        m1.descr_append(self.space, w_b)
+        assert m2.instantiation_count == count + 1
diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -800,7 +800,7 @@
         elif self.nextmap_first:
             self.nextmap_first._check_invariants()
 
-    def get_next(self, w_key, string, start, stop, terminator):
+    def get_next(self, w_key, string, start, stop, terminator=None):
         """ Returns the next map, given a wrapped key w_key, the json input
         string with positions start and stop, as well as a terminator.
 
@@ -843,6 +843,8 @@
             # one new leaf has been created
             self.change_number_of_leaves(1)
 
+        if terminator is None: # only from descr_append
+            terminator = self._slowly_get_terminator()
         terminator.register_potential_fringe(next)
         return next
 
@@ -869,7 +871,7 @@
                 return nextmap_first
         return None
 
-    def observe_transition(self, newmap, terminator):
+    def observe_transition(self, newmap, terminator=None):
         """ observe a transition from self to newmap.
         This does a few things, including updating the self size estimate with
         the knowledge that one object transitioned from self to newmap.
@@ -877,6 +879,8 @@
         newmap.instantiation_count += 1
         if isinstance(self, JSONMap) and self.state == MapBase.FRINGE:
             if self.is_useful():
+                if terminator is None:
+                    self._slowly_get_terminator()
                 self.mark_useful(terminator)
 
     def _make_next_map(self, w_key, key_repr):
@@ -899,21 +903,43 @@
     # exposed methods
 
     def descr_instantiate_dict(self, space, w_l):
-        from pypy.objspace.std.jsondict import from_values_and_jsonmap
+        """ Create a dict instance given from the structure, using the list l
+        as values. It must have the same length as the structure has keys. """
+        from pypy.objspace.std.jsondict import from_values_and_jsonmap_checked
         from pypy.objspace.std.jsondict import devolve_jsonmap_dict
         l_w = space.listview(w_l)
         if not isinstance(self, JSONMap):
             return space.newdict()
-        keys_w = self.get_keys_in_order()
-        if len(l_w) != len(keys_w):
-            raise oefmt(space.w_ValueError, "expected %s values, got %s", str(len(keys_w)), str(len(l_w)))
-        w_dict = from_values_and_jsonmap(self.space, l_w[:], self)
+        w_dict = from_values_and_jsonmap_checked(self.space, l_w[:], self)
         if self.is_state_blocked():
             devolve_jsonmap_dict(w_dict)
         return w_dict
 
     def descr_repr(self, space):
-        return space.newtext("<DictStructure [%s]>" % ", ".join([space.text_w(space.repr(w_key)) for w_key in self.get_keys_in_order()]))
+        res = []
+        curr = self
+        while type(curr) is JSONMap:
+            res.append(space.text_w(space.repr(self.w_key)))
+            curr = curr.prev
+        res.reverse()
+        return space.newtext("<DictStructure [%s]>" % ", ".join(res))
+
+    def descr_append(self, space, w_u):
+        from pypy.objspace.std.unicodeobject import W_UnicodeObject
+        if type(w_u) is not W_UnicodeObject:
+            raise oefmt(space.w_TypeError, "expected unicode, got %T", w_u)
+        u = space.utf8_w(w_u)
+        res = self.get_next(w_u, u, 0, len(u))
+        if res is None:
+            raise oefmt(space.w_ValueError, "repeated key %R", w_u)
+        self.observe_transition(res)
+        return res
+
+    def _slowly_get_terminator(self):
+        curr = self
+        while type(curr) is JSONMap:
+            curr = curr.prev
+        return curr
 
 
     # ____________________________________________________________
@@ -1190,5 +1216,6 @@
     __repr__ = interp2app(MapBase.descr_repr),
     last_key = GetSetProperty(get_last_key, name="last_key"),
     previous = GetSetProperty(get_previous, name="previous"),
+    append = interp2app(MapBase.descr_append),
 )
 MapBase.typedef.acceptable_as_base_class = False
diff --git a/pypy/objspace/std/jsondict.py b/pypy/objspace/std/jsondict.py
--- a/pypy/objspace/std/jsondict.py
+++ b/pypy/objspace/std/jsondict.py
@@ -8,7 +8,7 @@
 from pypy.objspace.std.dictmultiobject import (
     UnicodeDictStrategy, DictStrategy,
     create_iterator_classes, W_DictObject)
-
+from pypy.interpreter.error import oefmt
 
 def from_values_and_jsonmap(space, values_w, jsonmap):
     debug.make_sure_not_resized(values_w)
@@ -21,6 +21,17 @@
     storage = strategy.erase(values_w)
     return W_DictObject(space, strategy, storage)
 
+def from_values_and_jsonmap_checked(space, values_w, jsonmap):
+    debug.make_sure_not_resized(values_w)
+    strategy = jsonmap.strategy_instance
+    if strategy is None:
+        jsonmap.strategy_instance = strategy = JsonDictStrategy(space, jsonmap)
+    keys_w = strategy.get_keys_in_order()
+    if len(values_w) != len(keys_w):
+        raise oefmt(space.w_ValueError, "expected %s values, got %s", str(len(keys_w)), str(len(values_w)))
+    storage = strategy.erase(values_w)
+    return W_DictObject(space, strategy, storage)
+
 def devolve_jsonmap_dict(w_dict):
     assert isinstance(w_dict, W_DictObject)
     strategy = w_dict.get_strategy()

From pypy.commits at gmail.com  Mon Nov  4 07:26:24 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 04 Nov 2019 04:26:24 -0800 (PST)
Subject: [pypy-commit] buildbot default: tweak environment variables
Message-ID: <5dc018f0.1c69fb81.48cc.a553@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r1101:4f86f041b38d
Date: 2019-11-04 07:26 -0500
http://bitbucket.org/pypy/buildbot/changeset/4f86f041b38d/

Log:	tweak environment variables

diff --git a/docker/Dockerfile b/docker/Dockerfile
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -56,12 +56,12 @@
 
 # build a python 2.7 virtualenv, use the scl-installed version of python2.7
 # since centos6 itself has python2.6
-ENV LD_LIBRARY_PATH=/opt/rh/python27/root/usr/lib64
+ENV LD_LIBRARY_PATH=/opt/rh/python27/root/usr/lib64:/usr/local/lib64
 RUN /opt/rh/python27/root/usr/bin/python -mvirtualenv /python27_virt
 ENV PATH=/python27_virt/bin:$PATH
 RUN pip install --upgrade pip setuptools
 RUN pip install buildbot-slave pytest hypothesis cffi vmprof mercurial
-ENV PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig
+ENV PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
 
 CMD if [ -e /build_dir/buildbot.tac ]; then \
     buildslave start --nodaemon /build_dir; \

From pypy.commits at gmail.com  Mon Nov  4 07:27:31 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 04 Nov 2019 04:27:31 -0800 (PST)
Subject: [pypy-commit] pypy default: tweak packaging to avoid automatic
 platform detection, add more debug info
Message-ID: <5dc01933.1c69fb81.a2525.d49e@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r97950:976f9b5deb96
Date: 2019-11-04 07:09 -0500
http://bitbucket.org/pypy/pypy/changeset/976f9b5deb96/

Log:	tweak packaging to avoid automatic platform detection, add more
	debug info

diff --git a/pypy/tool/release/make_portable.py b/pypy/tool/release/make_portable.py
--- a/pypy/tool/release/make_portable.py
+++ b/pypy/tool/release/make_portable.py
@@ -73,7 +73,7 @@
 def make_portable():
     binaries = glob('bin/libpypy*.so')
     if not binaries:
-        raise ValueError('Could not find bin/libpypy*.so')
+        raise ValueError('Could not find bin/libpypy*.so in "%s"' % os.getcwd())
     binaries.extend(glob('lib_pypy/*_cffi.pypy*.so'))
     binaries.extend(glob('lib_pypy/_pypy_openssl*.so'))
     binaries.extend(glob('lib_pypy/_tkinter/*_cffi.pypy*.so'))
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -239,7 +239,11 @@
             # make the package portable by adding rpath=$ORIGIN/..lib,
             # bundling dependencies
             if options.make_portable:
+                os.chdir(str(name))
+                if not os.path.exists('lib'):
+                    os.mkdir('lib')
                 make_portable()
+                os.chdir(str(builddir))
         if USE_ZIPFILE_MODULE:
             import zipfile
             archive = str(builddir.join(name + '.zip'))
@@ -323,11 +327,10 @@
                         default=(sys.platform == 'darwin'),
                         help='whether to embed dependencies in CFFI modules '
                         '(default on OS X)')
-    parser.add_argument('--make-portable', '--no-make-portable',
+    parser.add_argument('--make-portable',
                         dest='make_portable',
-                        action=NegateAction,
-                        default=(platform.linux_distribution() in ('CentOS',)),
-                        help='whether to make the package portable by shipping '
+                        action='store_true',
+                        help='make the package portable by shipping '
                             'dependent shared objects and mangling RPATH')
     options = parser.parse_args(args)
 
@@ -340,9 +343,7 @@
     elif os.environ.has_key("PYPY_NO_EMBED_DEPENDENCIES"):
         options.embed_dependencies = False
     if os.environ.has_key("PYPY_MAKE_PORTABLE"):
-        options.embed_dependencies = True
-    elif os.environ.has_key("PYPY_NO_MAKE_PORTABLE"):
-        options.embed_dependencies = False
+        options.make_portable = True
     if not options.builddir:
         # The import actually creates the udir directory
         from rpython.tool.udir import udir

From pypy.commits at gmail.com  Mon Nov  4 12:58:06 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 04 Nov 2019 09:58:06 -0800 (PST)
Subject: [pypy-commit] pypy default: tweak _curses_build.py to fill in
 version, quiet cffi warnings
Message-ID: <5dc066ae.1c69fb81.eb894.d8a8@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r97951:3d6ba54d426e
Date: 2019-11-04 12:55 -0500
http://bitbucket.org/pypy/pypy/changeset/3d6ba54d426e/

Log:	tweak _curses_build.py to fill in version, quiet cffi warnings

diff --git a/lib_pypy/_curses_build.py b/lib_pypy/_curses_build.py
--- a/lib_pypy/_curses_build.py
+++ b/lib_pypy/_curses_build.py
@@ -1,4 +1,4 @@
-from cffi import FFI
+from cffi import FFI, VerificationError
 import os
 
 version_str = '''
@@ -6,13 +6,18 @@
     static const int NCURSES_VERSION_MINOR;
 '''
 
+version = (0, 0)
 def find_library(options):
+    global version
     for library in options:
         ffi = FFI()
         ffi.cdef(version_str)
         ffi.set_source("_curses_cffi_check", version_str, libraries=[library])
         try:
-            lib = ffi.compile()
+            ffi.compile()
+            import _curses_cffi_check
+            lib = _curses_cffi_check.lib
+            version = (lib.NCURSES_VERSION_MAJOR, lib.NCURSES_VERSION_MINOR)
         except VerificationError as e:
             e_last = e
             continue
@@ -82,9 +87,6 @@
                 find_library(['panel', 'panelw'])],
      include_dirs=find_curses_include_dirs())
 
-import _curses_cffi_check
-lib = _curses_cffi_check.lib
-version = (lib.NCURSES_VERSION_MAJOR, lib.NCURSES_VERSION_MINOR)
 
 ffi.cdef("""
 typedef ... WINDOW;
@@ -167,11 +169,11 @@
 
 int setupterm(char *, int, int *);
 
-WINDOW *stdscr;
-int COLORS;
-int COLOR_PAIRS;
-int COLS;
-int LINES;
+extern WINDOW *stdscr;
+extern int COLORS;
+extern int COLOR_PAIRS;
+extern int COLS;
+extern int LINES;
 
 int baudrate(void);
 int beep(void);
@@ -347,7 +349,7 @@
 #define _m_NetBSD ...
 int _m_ispad(WINDOW *);
 
-chtype acs_map[];
+extern chtype acs_map[];
 
 // For _curses_panel:
 

From pypy.commits at gmail.com  Mon Nov  4 12:58:07 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 04 Nov 2019 09:58:07 -0800 (PST)
Subject: [pypy-commit] pypy py3.6: merge default into py3.6
Message-ID: <5dc066af.1c69fb81.b8a04.18a0@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r97952:05b20def3304
Date: 2019-11-04 12:55 -0500
http://bitbucket.org/pypy/pypy/changeset/05b20def3304/

Log:	merge default into py3.6

diff --git a/lib_pypy/_curses_build.py b/lib_pypy/_curses_build.py
--- a/lib_pypy/_curses_build.py
+++ b/lib_pypy/_curses_build.py
@@ -1,4 +1,4 @@
-from cffi import FFI
+from cffi import FFI, VerificationError
 import os
 
 version_str = '''
@@ -6,13 +6,18 @@
     static const int NCURSES_VERSION_MINOR;
 '''
 
+version = (0, 0)
 def find_library(options):
+    global version
     for library in options:
         ffi = FFI()
         ffi.cdef(version_str)
         ffi.set_source("_curses_cffi_check", version_str, libraries=[library])
         try:
-            lib = ffi.compile()
+            ffi.compile()
+            import _curses_cffi_check
+            lib = _curses_cffi_check.lib
+            version = (lib.NCURSES_VERSION_MAJOR, lib.NCURSES_VERSION_MINOR)
         except VerificationError as e:
             e_last = e
             continue
@@ -82,9 +87,6 @@
                 find_library(['panel', 'panelw'])],
      include_dirs=find_curses_include_dirs())
 
-import _curses_cffi_check
-lib = _curses_cffi_check.lib
-version = (lib.NCURSES_VERSION_MAJOR, lib.NCURSES_VERSION_MINOR)
 
 ffi.cdef("""
 typedef ... WINDOW;
@@ -167,11 +169,11 @@
 
 int setupterm(char *, int, int *);
 
-WINDOW *stdscr;
-int COLORS;
-int COLOR_PAIRS;
-int COLS;
-int LINES;
+extern WINDOW *stdscr;
+extern int COLORS;
+extern int COLOR_PAIRS;
+extern int COLS;
+extern int LINES;
 
 int baudrate(void);
 int beep(void);
@@ -347,7 +349,7 @@
 #define _m_NetBSD ...
 int _m_ispad(WINDOW *);
 
-chtype acs_map[];
+extern chtype acs_map[];
 
 // For _curses_panel:
 
diff --git a/pypy/tool/release/make_portable.py b/pypy/tool/release/make_portable.py
--- a/pypy/tool/release/make_portable.py
+++ b/pypy/tool/release/make_portable.py
@@ -73,7 +73,7 @@
 def make_portable():
     binaries = glob('bin/libpypy*.so')
     if not binaries:
-        raise ValueError('Could not find bin/libpypy*.so')
+        raise ValueError('Could not find bin/libpypy*.so in "%s"' % os.getcwd())
     binaries.extend(glob('lib_pypy/*_cffi.pypy*.so'))
     binaries.extend(glob('lib_pypy/_pypy_openssl*.so'))
     binaries.extend(glob('lib_pypy/_tkinter/*_cffi.pypy*.so'))
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -248,7 +248,11 @@
             # make the package portable by adding rpath=$ORIGIN/..lib,
             # bundling dependencies
             if options.make_portable:
+                os.chdir(str(name))
+                if not os.path.exists('lib'):
+                    os.mkdir('lib')
                 make_portable()
+                os.chdir(str(builddir))
         if USE_ZIPFILE_MODULE:
             import zipfile
             archive = str(builddir.join(name + '.zip'))
@@ -332,11 +336,10 @@
                         default=(sys.platform == 'darwin'),
                         help='whether to embed dependencies in CFFI modules '
                         '(default on OS X)')
-    parser.add_argument('--make-portable', '--no-make-portable',
+    parser.add_argument('--make-portable',
                         dest='make_portable',
-                        action=NegateAction,
-                        default=(platform.linux_distribution() in ('CentOS',)),
-                        help='whether to make the package portable by shipping '
+                        action='store_true',
+                        help='make the package portable by shipping '
                             'dependent shared objects and mangling RPATH')
     options = parser.parse_args(args)
 
@@ -349,9 +352,7 @@
     elif os.environ.has_key("PYPY_NO_EMBED_DEPENDENCIES"):
         options.embed_dependencies = False
     if os.environ.has_key("PYPY_MAKE_PORTABLE"):
-        options.embed_dependencies = True
-    elif os.environ.has_key("PYPY_NO_MAKE_PORTABLE"):
-        options.embed_dependencies = False
+        options.make_portable = True
     if not options.builddir:
         # The import actually creates the udir directory
         from rpython.tool.udir import udir

From pypy.commits at gmail.com  Mon Nov  4 13:34:08 2019
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 04 Nov 2019 10:34:08 -0800 (PST)
Subject: [pypy-commit] pypy py3.6: Fix range of allowed years to match
 CPython (see bpo-13312)
Message-ID: <5dc06f20.1c69fb81.5d599.066a@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6
Changeset: r97953:edba5d677359
Date: 2019-11-04 18:33 +0000
http://bitbucket.org/pypy/pypy/changeset/edba5d677359/

Log:	Fix range of allowed years to match CPython (see bpo-13312)

diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -10,7 +10,8 @@
     str_decode_locale_surrogateescape, unicode_encode_locale_surrogateescape)
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rlib.rarithmetic import (
-    intmask, r_ulonglong, r_longfloat, widen, ovfcheck, ovfcheck_float_to_int)
+    intmask, r_ulonglong, r_longfloat, widen, ovfcheck, ovfcheck_float_to_int,
+    INT_MIN)
 from rpython.rlib.rtime import (GETTIMEOFDAY_NO_TZ, TIMEVAL,
                                 HAVE_GETTIMEOFDAY, HAVE_FTIME)
 from rpython.rlib import rposix, rtime
@@ -593,6 +594,8 @@
                     len(tup_w))
 
     y = space.c_int_w(tup_w[0])
+    if y < INT_MIN + 1900:
+        raise oefmt(space.w_OverflowError, "year out of range")
     tm_mon = space.c_int_w(tup_w[1])
     if tm_mon == 0:
         tm_mon = 1
diff --git a/pypy/module/time/test/test_time.py b/pypy/module/time/test/test_time.py
--- a/pypy/module/time/test/test_time.py
+++ b/pypy/module/time/test/test_time.py
@@ -153,6 +153,17 @@
         else:
             assert time.ctime(res) == 'Sat Jan  1 00:00:00 2000'
 
+    def test_mktime_overflow(self):
+        import time
+        MAX_YEAR = (1 << 31) - 1
+        MIN_YEAR = -(1 << 31) + 1900
+        time.mktime((MAX_YEAR,) + (0,) * 8)  # doesn't raise
+        with raises(OverflowError):
+            time.mktime((MAX_YEAR + 1,) + (0,) * 8)
+        time.mktime((MIN_YEAR,) + (0,) * 8)  # doesn't raise
+        with raises(OverflowError):
+            time.mktime((MIN_YEAR - 1,) + (0,) * 8)
+
     def test_asctime(self):
         import time
         time.asctime()

From pypy.commits at gmail.com  Mon Nov  4 15:11:05 2019
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 04 Nov 2019 12:11:05 -0800 (PST)
Subject: [pypy-commit] pypy py3.6: Handle large sys.tracebacklimit correctly
Message-ID: <5dc085d9.1c69fb81.398d5.d697@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6
Changeset: r97954:7d23f033e2ba
Date: 2019-11-04 20:10 +0000
http://bitbucket.org/pypy/pypy/changeset/7d23f033e2ba/

Log:	Handle large sys.tracebacklimit correctly

diff --git a/pypy/module/sys/app.py b/pypy/module/sys/app.py
--- a/pypy/module/sys/app.py
+++ b/pypy/module/sys/app.py
@@ -31,6 +31,8 @@
             # one is counting from the top, the other from the bottom of the
             # stack. so reverse polarity here
             if limit > 0:
+                if limit > sys.maxsize:
+                    limit = sys.maxsize
                 print_exception(exctype, value, traceback, limit=-limit)
             else:
                 # the limit is 0 or negative. PyTraceBack_Print does not print
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -163,7 +163,7 @@
             # see comment in 'setup_after_space_initialization'
             untranslated_enc = {'win32': 'mbcs', 'darwin': 'utf-8'}.get(enc, 'ascii')
             assert enc == untranslated_enc
-            
+
 
     def test_float_info(self):
         import sys
@@ -402,9 +402,9 @@
             raise ValueError(42)
 
         def get_error_with_tracebacklimit(limit):
-            import io
+            import _io
             sys.tracebacklimit = limit
-            sys.stderr = err = io.StringIO()
+            sys.stderr = err = _io.StringIO()
             try:
                 f1()
             except ValueError:
@@ -425,6 +425,12 @@
         assert "Traceback (most recent call last):" not in msg
         assert "ValueError" in msg
 
+        msg = get_error_with_tracebacklimit(1<<100)
+        assert "Traceback (most recent call last):" in msg
+        assert "f1" in msg
+        assert "f2" in msg
+        assert "f3" in msg
+
         sys.stderr = savestderr
         del sys.tracebacklimit
 

From pypy.commits at gmail.com  Tue Nov  5 11:17:57 2019
From: pypy.commits at gmail.com (arigo)
Date: Tue, 05 Nov 2019 08:17:57 -0800 (PST)
Subject: [pypy-commit] pypy py3.6: add the 3.6.1 new functions
 PySlice_Unpack() and PySlice_AdjustIndices()
Message-ID: <5dc1a0b5.1c69fb81.ad90.01e6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.6
Changeset: r97957:5b94ae9bcf6d
Date: 2019-11-05 17:12 +0100
http://bitbucket.org/pypy/pypy/changeset/5b94ae9bcf6d/

Log:	add the 3.6.1 new functions PySlice_Unpack() and
	PySlice_AdjustIndices()

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1528,6 +1528,7 @@
                          source_dir / "object.c",
                          source_dir / "typeobject.c",
                          source_dir / "tupleobject.c",
+                         source_dir / "sliceobject.c",
                          ]
 
 def build_eci(code, use_micronumpy=False, translating=False):
diff --git a/pypy/module/cpyext/include/sliceobject.h b/pypy/module/cpyext/include/sliceobject.h
--- a/pypy/module/cpyext/include/sliceobject.h
+++ b/pypy/module/cpyext/include/sliceobject.h
@@ -18,7 +18,10 @@
 } PySliceObject;
 
 #define PySlice_Check(op) ((op)->ob_type == &PySlice_Type)
-    
+
+PyAPI_FUNC(Py_ssize_t) PySlice_AdjustIndices(Py_ssize_t length,
+                         Py_ssize_t *start, Py_ssize_t *stop, Py_ssize_t step);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/pypy/module/cpyext/sliceobject.py b/pypy/module/cpyext/sliceobject.py
--- a/pypy/module/cpyext/sliceobject.py
+++ b/pypy/module/cpyext/sliceobject.py
@@ -1,3 +1,4 @@
+import sys
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, cpython_struct, bootstrap_function, build_type_checkers,
@@ -5,7 +6,7 @@
 from pypy.module.cpyext.pyobject import (
     decref, PyObject, make_ref, make_typedescr)
 from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import oefmt
 from pypy.objspace.std.sliceobject import W_SliceObject
 
 # Slice objects directly expose their members as PyObject.
@@ -75,7 +76,7 @@
 
     Returns 0 on success and -1 on error with exception set."""
     if not isinstance(w_slice, W_SliceObject):
-        PyErr_BadInternalCall(space)
+        raise PyErr_BadInternalCall(space)
     start_p[0], stop_p[0], step_p[0], slicelength_p[0] = \
             w_slice.indices4(space, length)
     return 0
@@ -96,7 +97,37 @@
     incorporate the source of PySlice_GetIndicesEx(), suitably renamed,
     in the source of your extension."""
     if not isinstance(w_slice, W_SliceObject):
-        PyErr_BadInternalCall(space)
+        raise PyErr_BadInternalCall(space)
     start_p[0], stop_p[0], step_p[0] = \
             w_slice.indices3(space, length)
     return 0
+
+ at cpython_api([PyObject, Py_ssize_tP, Py_ssize_tP, Py_ssize_tP],
+             rffi.INT_real, error=-1)
+def PySlice_Unpack(space, w_slice, start_p, stop_p, step_p):
+    if not isinstance(w_slice, W_SliceObject):
+        raise PyErr_BadInternalCall(space)
+
+    if space.is_none(w_slice.w_step):
+        step = 1
+    else:
+        step = W_SliceObject.eval_slice_index(space, w_slice.w_step)
+        if step == 0:
+            raise oefmt(space.w_ValueError, "slice step cannot be zero")
+        if step < -sys.maxint:
+            step = -sys.maxint
+    step_p[0] = step
+
+    if space.is_none(w_slice.w_start):
+        start = sys.maxint if step < 0 else 0
+    else:
+        start = W_SliceObject.eval_slice_index(space, w_slice.w_start)
+    start_p[0] = start
+
+    if space.is_none(w_slice.w_stop):
+        stop = -sys.maxint-1 if step < 0 else sys.maxint
+    else:
+        stop = W_SliceObject.eval_slice_index(space, w_slice.w_stop)
+    stop_p[0] = stop
+
+    return 0
diff --git a/pypy/module/cpyext/src/sliceobject.c b/pypy/module/cpyext/src/sliceobject.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/src/sliceobject.c
@@ -0,0 +1,43 @@
+#include "Python.h"
+
+Py_ssize_t
+PySlice_AdjustIndices(Py_ssize_t length,
+                      Py_ssize_t *start, Py_ssize_t *stop, Py_ssize_t step)
+{
+    /* this is harder to get right than you might think */
+
+    assert(step != 0);
+    assert(step >= -PY_SSIZE_T_MAX);
+
+    if (*start < 0) {
+        *start += length;
+        if (*start < 0) {
+            *start = (step < 0) ? -1 : 0;
+        }
+    }
+    else if (*start >= length) {
+        *start = (step < 0) ? length - 1 : length;
+    }
+
+    if (*stop < 0) {
+        *stop += length;
+        if (*stop < 0) {
+            *stop = (step < 0) ? -1 : 0;
+        }
+    }
+    else if (*stop >= length) {
+        *stop = (step < 0) ? length - 1 : length;
+    }
+
+    if (step < 0) {
+        if (*stop < *start) {
+            return (*start - *stop - 1) / (-step) + 1;
+        }
+    }
+    else {
+        if (*start < *stop) {
+            return (*stop - *start - 1) / step + 1;
+        }
+    }
+    return 0;
+}
diff --git a/pypy/module/cpyext/test/test_sliceobject.py b/pypy/module/cpyext/test/test_sliceobject.py
--- a/pypy/module/cpyext/test/test_sliceobject.py
+++ b/pypy/module/cpyext/test/test_sliceobject.py
@@ -1,3 +1,4 @@
+import sys
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
@@ -84,3 +85,47 @@
             ])
         s = slice(10, 20, 30)
         assert module.check(s)
+
+    def test_Unpack(self):
+        from sys import maxsize as M
+        module = self.import_extension('foo', [
+            ("check", "METH_O",
+             """
+                 Py_ssize_t start, stop, step;
+                 if (PySlice_Unpack(args, &start, &stop, &step) != 0)
+                     return NULL;
+                 return Py_BuildValue("nnn", start, stop, step);
+             """),
+            ])
+        assert module.check(slice(10, 20, 1)) == (10, 20, 1)
+        assert module.check(slice(None, 20, 1)) == (0, 20, 1)
+        assert module.check(slice(10, None, 3)) == (10, M, 3)
+        assert module.check(slice(10, 20, None)) == (10, 20, 1)
+        assert module.check(slice(20, 5, 1)) == (20, 5, 1)
+        assert module.check(slice(None, None, None)) == (0, M, 1)
+
+        assert module.check(slice(20, 10, -1)) == (20, 10, -1)
+        assert module.check(slice(None, 20, -1)) == (M, 20, -1)
+        assert module.check(slice(10, None, -1)) == (10, -M-1, -1)
+
+        assert module.check(slice(M*2, M*3, 1)) == (M, M, 1)
+        assert module.check(slice(M*2, -123, 1)) == (M, -123, 1)
+        assert module.check(slice(-M*2, -M*3, 1)) == (-M-1, -M-1, 1)
+        assert module.check(slice(-M*2, 123, -2)) == (-M-1, 123, -2)
+
+        with raises(ValueError):
+            module.check(slice(2, 3, 0))
+        assert module.check(slice(2, 3, -M-1)) == (2, 3, -M)
+        assert module.check(slice(2, 3, -M-10)) == (2, 3, -M)
+        assert module.check(slice(2, 3, M+10)) == (2, 3, M)
+
+    def test_AdjustIndices(self):
+        module = self.import_extension('foo', [
+            ("check", "METH_NOARGS",
+             """
+                 Py_ssize_t start = -35, stop = 99999, step = 10, result;
+                 result = PySlice_AdjustIndices(100, &start, &stop, step);
+                 return Py_BuildValue("nnnn", result, start, stop, step);
+             """),
+            ])
+        assert module.check() == (4, 65, 100, 10)
diff --git a/pypy/objspace/std/sliceobject.py b/pypy/objspace/std/sliceobject.py
--- a/pypy/objspace/std/sliceobject.py
+++ b/pypy/objspace/std/sliceobject.py
@@ -172,6 +172,11 @@
         return app_indices(space, self.w_start, self.w_stop,
                            self.w_step, w_length)
 
+    @staticmethod
+    def eval_slice_index(space, w_int):
+        """Helper for cpyext"""
+        return _eval_slice_index(space, w_int)
+
 
 def slicewprop(name):
     def fget(space, w_obj):

From pypy.commits at gmail.com  Tue Nov  5 14:24:24 2019
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 05 Nov 2019 11:24:24 -0800 (PST)
Subject: [pypy-commit] pypy py3.6: fix test_flufl
Message-ID: <5dc1cc68.1c69fb81.ff9a7.1ebe@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6
Changeset: r97962:fa561c2cb364
Date: 2019-11-05 19:23 +0000
http://bitbucket.org/pypy/pypy/changeset/fa561c2cb364/

Log:	fix test_flufl

diff --git a/lib-python/3/test/test_flufl.py b/lib-python/3/test/test_flufl.py
--- a/lib-python/3/test/test_flufl.py
+++ b/lib-python/3/test/test_flufl.py
@@ -15,7 +15,7 @@
         self.assertEqual(cm.exception.text, '2 != 3\n')
         self.assertEqual(cm.exception.filename, '<FLUFL test>')
         self.assertEqual(cm.exception.lineno, 2)
-        self.assertEqual(cm.exception.offset, 4)
+        self.assertEqual(cm.exception.offset, 2)  # changed in PyPy
 
     def test_guido_as_bdfl(self):
         code = '2 {0} 3'
@@ -26,7 +26,7 @@
         self.assertEqual(cm.exception.text, '2 <> 3\n')
         self.assertEqual(cm.exception.filename, '<FLUFL test>')
         self.assertEqual(cm.exception.lineno, 1)
-        self.assertEqual(cm.exception.offset, 4)
+        self.assertEqual(cm.exception.offset, 2)  # changed in PyPy
 
 
 if __name__ == '__main__':
diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -497,10 +497,10 @@
 
     def handle_async_funcdef(self, node, decorators=None):
         return self.handle_funcdef_impl(node.get_child(1), 1, decorators)
-    
+
     def handle_funcdef(self, node, decorators=None):
         return self.handle_funcdef_impl(node, 0, decorators)
-    
+
     def handle_async_stmt(self, node):
         ch = node.get_child(1)
         if ch.type == syms.funcdef:
@@ -942,7 +942,7 @@
                 if flufl and comp_node.get_value() == '!=':
                     self.error("with Barry as BDFL, use '<>' instead of '!='", comp_node)
                 elif not flufl and comp_node.get_value() == '<>':
-                    self.error('invalid comparison', comp_node)
+                    self.error('invalid syntax', comp_node)
                 return ast.NotEq
             elif comp_type == tokens.NAME:
                 if comp_node.get_value() == "is":
@@ -1014,7 +1014,7 @@
                              atom_node.get_column())
         else:
             return atom_expr
-    
+
     def handle_power(self, power_node):
         atom_expr = self.handle_atom_expr(power_node.get_child(0))
         if power_node.num_children() == 1:
@@ -1092,7 +1092,7 @@
     def handle_call(self, args_node, callable_expr):
         arg_count = 0 # position args + iterable args unpackings
         keyword_count = 0 # keyword args + keyword args unpackings
-        generator_count = 0 
+        generator_count = 0
         for i in range(args_node.num_children()):
             argument = args_node.get_child(i)
             if argument.type == syms.argument:
@@ -1300,7 +1300,6 @@
                     if is_dict:
                         raise self.error("dict unpacking cannot be used in "
                                          "dict comprehension", atom_node)
-                    
                     return self.handle_dictcomp(maker, atom_node)
                 else:
                     # a dictionary display
@@ -1423,7 +1422,7 @@
         comps = self.comprehension_helper(dict_maker.get_child(i))
         return ast.DictComp(key, value, comps, atom_node.get_lineno(),
                                                atom_node.get_column())
-    
+
     def handle_dictdisplay(self, node, atom_node):
         keys = []
         values = []
@@ -1435,7 +1434,7 @@
             i += 1
         return ast.Dict(keys, values, atom_node.get_lineno(),
                                       atom_node.get_column())
-    
+
     def handle_setdisplay(self, node, atom_node):
         elts = []
         i = 0

From pypy.commits at gmail.com  Wed Nov  6 04:01:18 2019
From: pypy.commits at gmail.com (stevie_92)
Date: Wed, 06 Nov 2019 01:01:18 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-gc-cycle: Increased incremental limit of
 rrc incmark
Message-ID: <5dc28bde.1c69fb81.5a3c9.bbc0@mx.google.com>

Author: Stefan Beyer <home at sbeyer.at>
Branch: cpyext-gc-cycle
Changeset: r97967:1fd7c207819e
Date: 2019-11-06 10:00 +0100
http://bitbucket.org/pypy/pypy/changeset/1fd7c207819e/

Log:	Increased incremental limit of rrc incmark Changed default rrc
	implementation Added simple rrc implementation (to mimic old
	implementation)

diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -106,8 +106,8 @@
                                 ("translation.backend", "c")],
                     }),
     ChoiceOption("rrcgc", "Garbage Collection Strategy for raw refcounted objects in cpyext",
-                 ["mark", "incmark", "none"],
-                 default="mark",
+                 ["simple", "mark", "incmark"],
+                 default="incmark",
                  requires={
                     "mark": [("translation.gc", "incminimark")],
                     "incmark": [("translation.gc", "incminimark")],
diff --git a/rpython/memory/gc/rrc/base.py b/rpython/memory/gc/rrc/base.py
--- a/rpython/memory/gc/rrc/base.py
+++ b/rpython/memory/gc/rrc/base.py
@@ -5,7 +5,8 @@
 
 def choose_rrc_gc_from_config(config):
     if config.translation.rrcgc:
-        classes = {"mark": "mark.RawRefCountMarkGC",
+        classes = {"simple": "simple.RawRefCountSimpleGC",
+                   "mark": "mark.RawRefCountMarkGC",
                    "incmark": "incmark.RawRefCountIncMarkGC",
                    }
         try:
@@ -126,7 +127,7 @@
         if inc_limit > 0:
             self.inc_limit = inc_limit
         else:
-            self.inc_limit = 1000
+            self.inc_limit = 50000
         self.cycle_enabled = True
 
     def create_link_pypy(self, gcobj, pyobject):
diff --git a/rpython/memory/gc/rrc/simple.py b/rpython/memory/gc/rrc/simple.py
new file mode 100644
--- /dev/null
+++ b/rpython/memory/gc/rrc/simple.py
@@ -0,0 +1,7 @@
+from rpython.memory.gc.rrc.base import RawRefCountBaseGC
+
+class RawRefCountSimpleGC(RawRefCountBaseGC):
+
+    def major_collection_trace_step(self):
+        self.p_list_old.foreach(self._major_trace, (False, False))
+        return True
\ No newline at end of file

From pypy.commits at gmail.com  Wed Nov  6 11:02:19 2019
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 06 Nov 2019 08:02:19 -0800 (PST)
Subject: [pypy-commit] pypy py3.6: Adjust test for changes to co_consts
Message-ID: <5dc2ee8b.1c69fb81.d5a7c.1736@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6
Changeset: r97970:7b678cb3bf52
Date: 2019-11-06 16:00 +0000
http://bitbucket.org/pypy/pypy/changeset/7b678cb3bf52/

Log:	Adjust test for changes to co_consts

diff --git a/lib-python/3/test/test_dis.py b/lib-python/3/test/test_dis.py
--- a/lib-python/3/test/test_dis.py
+++ b/lib-python/3/test/test_dis.py
@@ -686,9 +686,9 @@
 # End fodder for opinfo generation tests
 expected_outer_line = 1
 _line_offset = outer.__code__.co_firstlineno - 1
-code_object_f = outer.__code__.co_consts[3]
+code_object_f = outer.__code__.co_consts[2]
 expected_f_line = code_object_f.co_firstlineno - _line_offset
-code_object_inner = code_object_f.co_consts[3]
+code_object_inner = code_object_f.co_consts[2]
 expected_inner_line = code_object_inner.co_firstlineno - _line_offset
 expected_jumpy_line = 1
 
@@ -713,22 +713,22 @@
 
 Instruction = dis.Instruction
 expected_opinfo_outer = [
-  Instruction(opname='LOAD_CONST', opcode=100, arg=8, argval=(3, 4), argrepr='(3, 4)', offset=0, starts_line=2, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=1, argval=(3, 4), argrepr='(3, 4)', offset=0, starts_line=2, is_jump_target=False),
   Instruction(opname='LOAD_CLOSURE', opcode=135, arg=0, argval='a', argrepr='a', offset=2, starts_line=None, is_jump_target=False),
   Instruction(opname='LOAD_CLOSURE', opcode=135, arg=1, argval='b', argrepr='b', offset=4, starts_line=None, is_jump_target=False),
   Instruction(opname='BUILD_TUPLE', opcode=102, arg=2, argval=2, argrepr='', offset=6, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=code_object_f, argrepr=repr(code_object_f), offset=8, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='outer.<locals>.f', argrepr="'outer.<locals>.f'", offset=10, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=code_object_f, argrepr=repr(code_object_f), offset=8, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval='outer.<locals>.f', argrepr="'outer.<locals>.f'", offset=10, starts_line=None, is_jump_target=False),
   Instruction(opname='MAKE_FUNCTION', opcode=132, arg=9, argval=9, argrepr='', offset=12, starts_line=None, is_jump_target=False),
   Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='f', argrepr='f', offset=14, starts_line=None, is_jump_target=False),
   Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=16, starts_line=7, is_jump_target=False),
   Instruction(opname='LOAD_DEREF', opcode=136, arg=0, argval='a', argrepr='a', offset=18, starts_line=None, is_jump_target=False),
   Instruction(opname='LOAD_DEREF', opcode=136, arg=1, argval='b', argrepr='b', offset=20, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval='', argrepr="''", offset=22, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval=1, argrepr='1', offset=24, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='', argrepr="''", offset=22, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=24, starts_line=None, is_jump_target=False),
   Instruction(opname='BUILD_LIST', opcode=103, arg=0, argval=0, argrepr='', offset=26, starts_line=None, is_jump_target=False),
   Instruction(opname='BUILD_MAP', opcode=105, arg=0, argval=0, argrepr='', offset=28, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=7, argval='Hello world!', argrepr="'Hello world!'", offset=30, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval='Hello world!', argrepr="'Hello world!'", offset=30, starts_line=None, is_jump_target=False),
   Instruction(opname='CALL_FUNCTION', opcode=131, arg=7, argval=7, argrepr='', offset=32, starts_line=None, is_jump_target=False),
   Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=34, starts_line=None, is_jump_target=False),
   Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='f', argrepr='f', offset=36, starts_line=8, is_jump_target=False),
@@ -736,14 +736,14 @@
 ]
 
 expected_opinfo_f = [
-  Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=(5, 6), argrepr='(5, 6)', offset=0, starts_line=3, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=1, argval=(5, 6), argrepr='(5, 6)', offset=0, starts_line=3, is_jump_target=False),
   Instruction(opname='LOAD_CLOSURE', opcode=135, arg=2, argval='a', argrepr='a', offset=2, starts_line=None, is_jump_target=False),
   Instruction(opname='LOAD_CLOSURE', opcode=135, arg=3, argval='b', argrepr='b', offset=4, starts_line=None, is_jump_target=False),
   Instruction(opname='LOAD_CLOSURE', opcode=135, arg=0, argval='c', argrepr='c', offset=6, starts_line=None, is_jump_target=False),
   Instruction(opname='LOAD_CLOSURE', opcode=135, arg=1, argval='d', argrepr='d', offset=8, starts_line=None, is_jump_target=False),
   Instruction(opname='BUILD_TUPLE', opcode=102, arg=4, argval=4, argrepr='', offset=10, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=code_object_inner, argrepr=repr(code_object_inner), offset=12, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='outer.<locals>.f.<locals>.inner', argrepr="'outer.<locals>.f.<locals>.inner'", offset=14, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=code_object_inner, argrepr=repr(code_object_inner), offset=12, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval='outer.<locals>.f.<locals>.inner', argrepr="'outer.<locals>.f.<locals>.inner'", offset=14, starts_line=None, is_jump_target=False),
   Instruction(opname='MAKE_FUNCTION', opcode=132, arg=9, argval=9, argrepr='', offset=16, starts_line=None, is_jump_target=False),
   Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='inner', argrepr='inner', offset=18, starts_line=None, is_jump_target=False),
   Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=20, starts_line=5, is_jump_target=False),

From pypy.commits at gmail.com  Wed Nov  6 11:53:39 2019
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 06 Nov 2019 08:53:39 -0800 (PST)
Subject: [pypy-commit] pypy py3.6: adapt expected dis output to PyPy
Message-ID: <5dc2fa93.1c69fb81.2591f.41fb@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6
Changeset: r97971:baac25aa3936
Date: 2019-11-06 16:52 +0000
http://bitbucket.org/pypy/pypy/changeset/baac25aa3936/

Log:	adapt expected dis output to PyPy

diff --git a/lib-python/3/test/test_dis.py b/lib-python/3/test/test_dis.py
--- a/lib-python/3/test/test_dis.py
+++ b/lib-python/3/test/test_dis.py
@@ -262,7 +262,7 @@
              20 RETURN_VALUE
 """
 
-# XXX: change for PyPy?
+# changed for PyPy
 dis_traceback = """\
 %3d           0 SETUP_EXCEPT            12 (to 14)
 
@@ -280,18 +280,18 @@
              22 POP_TOP
              24 STORE_FAST               0 (e)
              26 POP_TOP
-             28 SETUP_FINALLY           12 (to 42)
+             28 SETUP_FINALLY           10 (to 40)
 
 %3d          30 LOAD_FAST                0 (e)
              32 LOAD_ATTR                1 (__traceback__)
              34 STORE_FAST               1 (tb)
              36 POP_BLOCK
-             38 POP_EXCEPT
-             40 LOAD_CONST               0 (None)
-        >>   42 LOAD_CONST               0 (None)
-             44 STORE_FAST               0 (e)
-             46 DELETE_FAST              0 (e)
-             48 END_FINALLY
+             38 LOAD_CONST               0 (None)
+        >>   40 LOAD_CONST               0 (None)
+             42 STORE_FAST               0 (e)
+             44 DELETE_FAST              0 (e)
+             46 END_FINALLY
+             48 POP_EXCEPT
              50 JUMP_FORWARD             2 (to 54)
         >>   52 END_FINALLY
 

From pypy.commits at gmail.com  Wed Nov  6 16:43:09 2019
From: pypy.commits at gmail.com (wlav)
Date: Wed, 06 Nov 2019 13:43:09 -0800 (PST)
Subject: [pypy-commit] pypy cppyy-packaging: closing old cppyy packaing
 branch to restart work
Message-ID: <5dc33e6d.1c69fb81.3faec.5ee3@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r97972:b9b63c51b979
Date: 2019-11-06 13:41 -0800
http://bitbucket.org/pypy/pypy/changeset/b9b63c51b979/

Log:	closing old cppyy packaing branch to restart work


From pypy.commits at gmail.com  Wed Nov  6 18:01:02 2019
From: pypy.commits at gmail.com (Yannick_Jadoul)
Date: Wed, 06 Nov 2019 15:01:02 -0800 (PST)
Subject: [pypy-commit] pypy py3.7-bpo-29962: Implemented app-level
 math.remainder
Message-ID: <5dc350ae.1c69fb81.2aa50.0332@mx.google.com>

Author: Yannick Jadoul <yannick.jadoul at belgacom.net>
Branch: py3.7-bpo-29962
Changeset: r97973:b1d5214588a0
Date: 2019-11-06 23:58 +0100
http://bitbucket.org/pypy/pypy/changeset/b1d5214588a0/

Log:	Implemented app-level math.remainder

diff --git a/pypy/module/math/app_math.py b/pypy/module/math/app_math.py
--- a/pypy/module/math/app_math.py
+++ b/pypy/module/math/app_math.py
@@ -47,3 +47,85 @@
     res, _, shift = _fac1(x)
     return res << shift
 
+
+def remainder(x, y):
+    """Difference between x and the closest integer multiple of y.
+
+    Return x - n*y where n*y is the closest integer multiple of y.
+    In the case where x is exactly halfway between two multiples of
+    y, the nearest even value of n is used. The result is always exact."""
+
+    from math import copysign, fabs, fmod, isfinite, isinf, isnan, nan
+
+    x = float(x)
+    y = float(y)
+
+    # Deal with most common case first.
+    if isfinite(x) and isfinite(y):
+        if y == 0.0:
+            # return nan
+            # Merging the logic from math_2 in CPython's mathmodule.c
+            # nan returned and x and y both not nan -> domain error
+            raise ValueError("math domain error")
+        
+        absx = fabs(x)
+        absy = fabs(y)
+        m = fmod(absx, absy)
+
+        # Warning: some subtlety here. What we *want* to know at this point is
+        # whether the remainder m is less than, equal to, or greater than half
+        # of absy. However, we can't do that comparison directly because we
+        # can't be sure that 0.5*absy is representable (the mutiplication
+        # might incur precision loss due to underflow). So instead we compare
+        # m with the complement c = absy - m: m < 0.5*absy if and only if m <
+        # c, and so on. The catch is that absy - m might also not be
+        # representable, but it turns out that it doesn't matter:
+        # - if m > 0.5*absy then absy - m is exactly representable, by
+        #     Sterbenz's lemma, so m > c
+        # - if m == 0.5*absy then again absy - m is exactly representable
+        #     and m == c
+        # - if m < 0.5*absy then either (i) 0.5*absy is exactly representable,
+        #     in which case 0.5*absy < absy - m, so 0.5*absy <= c and hence m <
+        #     c, or (ii) absy is tiny, either subnormal or in the lowest normal
+        #     binade. Then absy - m is exactly representable and again m < c.
+
+        c = absy - m
+        if m < c:
+            r = m
+        elif m > c:
+            r = -c
+        else:
+            # Here absx is exactly halfway between two multiples of absy,
+            # and we need to choose the even multiple. x now has the form
+            #     absx = n * absy + m
+            # for some integer n (recalling that m = 0.5*absy at this point).
+            # If n is even we want to return m; if n is odd, we need to
+            # return -m.
+            # So
+            #     0.5 * (absx - m) = (n/2) * absy
+            # and now reducing modulo absy gives us:
+            #                                     | m, if n is odd
+            #     fmod(0.5 * (absx - m), absy) = |
+            #                                     | 0, if n is even
+            # Now m - 2.0 * fmod(...) gives the desired result: m
+            # if n is even, -m if m is odd.
+            # Note that all steps in fmod(0.5 * (absx - m), absy)
+            # will be computed exactly, with no rounding error
+            # introduced.
+            assert m == c
+            r = m - 2.0 * fmod(0.5 * (absx - m), absy)
+        return copysign(1.0, x) * r
+    
+    # Special values.
+    if isnan(x):
+        return x
+    if isnan(y):
+        return y
+    if isinf(x):
+        # return nan
+        # Merging the logic from math_2 in CPython's mathmodule.c
+        # nan returned and x and y both not nan -> domain error
+        raise ValueError("math domain error")
+    assert isinf(y)
+    return x
+ 
\ No newline at end of file
diff --git a/pypy/module/math/moduledef.py b/pypy/module/math/moduledef.py
--- a/pypy/module/math/moduledef.py
+++ b/pypy/module/math/moduledef.py
@@ -5,6 +5,7 @@
 class Module(MixedModule):
     appleveldefs = {
        'factorial' : 'app_math.factorial',
+       'remainder' : 'app_math.remainder',
     }
 
     interpleveldefs = {
diff --git a/pypy/module/math/test/test_math.py b/pypy/module/math/test/test_math.py
--- a/pypy/module/math/test/test_math.py
+++ b/pypy/module/math/test/test_math.py
@@ -386,3 +386,17 @@
     def test_pi_tau(self):
         import math
         assert math.tau == math.pi * 2.0
+
+    def test_remainder(self):
+        import math
+        assert math.remainder(3, math.pi) == 3 - math.pi
+        assert math.remainder(-3, math.pi) == math.pi - 3
+        assert math.remainder(3, -math.pi) == 3 - math.pi
+        assert math.remainder(4, math.pi) == 4 - math.pi
+        assert math.remainder(6, math.pi) == 6 - 2 * math.pi
+        assert math.remainder(3, math.inf) == 3
+        assert math.remainder(3, -math.inf) == 3
+        assert math.isnan(math.remainder(3, math.nan))
+        assert math.isnan(math.remainder(math.nan, 3))
+        raises(ValueError, math.remainder, 3, 0)
+        raises(ValueError, math.remainder, math.inf, 3)

From pypy.commits at gmail.com  Thu Nov  7 04:29:18 2019
From: pypy.commits at gmail.com (arigo)
Date: Thu, 07 Nov 2019 01:29:18 -0800 (PST)
Subject: [pypy-commit] cffi default: Remove very old, unused file
Message-ID: <5dc3e3ee.1c69fb81.afe8b.6806@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3315:2d7c18be7ac6
Date: 2019-11-07 10:28 +0100
http://bitbucket.org/cffi/cffi/changeset/2d7c18be7ac6/

Log:	Remove very old, unused file

diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
--- a/.travis.yml
+++ /dev/null
@@ -1,12 +0,0 @@
-language: python
-python:
-    - "2.6"
-    - "2.7"
-    - "pypy"
-install:
-    - pip install .
-script: py.test
-notifications:
-    email: false
-    irc: "irc.freenode.org#pypy"
-

From pypy.commits at gmail.com  Thu Nov  7 06:24:57 2019
From: pypy.commits at gmail.com (arigo)
Date: Thu, 07 Nov 2019 03:24:57 -0800 (PST)
Subject: [pypy-commit] cffi default: Tweak the '-Wno-*' arguments passed to
 gcc during tests
Message-ID: <5dc3ff09.1c69fb81.42759.8ddb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3316:eebc6733b38d
Date: 2019-11-07 12:24 +0100
http://bitbucket.org/cffi/cffi/changeset/eebc6733b38d/

Log:	Tweak the '-Wno-*' arguments passed to gcc during tests

diff --git a/cffi/_cffi_include.h b/cffi/_cffi_include.h
--- a/cffi/_cffi_include.h
+++ b/cffi/_cffi_include.h
@@ -261,12 +261,12 @@
         return (int)_cffi_to_c_wchar3216_t(o);
 }
 
-_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char32_t(int x)
+_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char32_t(unsigned int x)
 {
     if (sizeof(_cffi_wchar_t) == 4)
         return _cffi_from_c_wchar_t((_cffi_wchar_t)x);
     else
-        return _cffi_from_c_wchar3216_t(x);
+        return _cffi_from_c_wchar3216_t((int)x);
 }
 
 
diff --git a/testing/cffi0/test_verify.py b/testing/cffi0/test_verify.py
--- a/testing/cffi0/test_verify.py
+++ b/testing/cffi0/test_verify.py
@@ -3,6 +3,7 @@
 import sys, os, math, weakref
 from cffi import FFI, VerificationError, VerificationMissing, model, FFIError
 from testing.support import *
+from testing.support import extra_compile_args
 
 
 lib_m = ['m']
@@ -13,17 +14,6 @@
         lib_m = ['msvcrt']
     pass      # no obvious -Werror equivalent on MSVC
 else:
-    if (sys.platform == 'darwin' and
-          [int(x) for x in os.uname()[2].split('.')] >= [11, 0, 0]):
-        # assume a standard clang or gcc
-        extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion']
-        # special things for clang
-        extra_compile_args.append('-Qunused-arguments')
-    else:
-        # assume a standard gcc
-        extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion',
-                              '-Wno-unused-parameter']
-
     class FFI(FFI):
         def verify(self, *args, **kwds):
             return super(FFI, self).verify(
diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -34,8 +34,9 @@
         source = 'extern "C" {\n%s\n}' % (source,)
     elif sys.platform != 'win32':
         # add '-Werror' to the existing 'extra_compile_args' flags
+        from testing.support import extra_compile_args
         kwds['extra_compile_args'] = (kwds.get('extra_compile_args', []) +
-                                      ['-Werror'])
+                                      extra_compile_args)
     return _verify(ffi, module_name, source, *args, **kwds)
 
 def test_set_source_no_slashes():
@@ -2038,7 +2039,7 @@
     ffi.cdef("float _Complex f1(float a, float b);");
     lib = verify(ffi, "test_function_returns_float_complex", """
         #include <complex.h>
-        static float _Complex f1(float a, float b) { return a + I*2.0*b; }
+        static float _Complex f1(float a, float b) { return a + I*2.0f*b; }
     """, no_cpp=True)    # <complex.h> fails on some systems with C++
     result = lib.f1(1.25, 5.1)
     assert type(result) == complex
diff --git a/testing/cffi1/test_verify1.py b/testing/cffi1/test_verify1.py
--- a/testing/cffi1/test_verify1.py
+++ b/testing/cffi1/test_verify1.py
@@ -4,7 +4,7 @@
 from cffi import CDefError
 from cffi import recompiler
 from testing.support import *
-from testing.support import _verify
+from testing.support import _verify, extra_compile_args
 import _cffi_backend
 
 lib_m = ['m']
@@ -13,18 +13,6 @@
     import distutils.ccompiler
     if distutils.ccompiler.get_default_compiler() == 'msvc':
         lib_m = ['msvcrt']
-    extra_compile_args = []      # no obvious -Werror equivalent on MSVC
-else:
-    if (sys.platform == 'darwin' and
-          [int(x) for x in os.uname()[2].split('.')] >= [11, 0, 0]):
-        # assume a standard clang or gcc
-        extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion']
-        # special things for clang
-        extra_compile_args.append('-Qunused-arguments')
-    else:
-        # assume a standard gcc
-        extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion',
-                              '-Wno-unused-parameter']
 
 class FFI(FFI):
     error = _cffi_backend.FFI.error
diff --git a/testing/support.py b/testing/support.py
--- a/testing/support.py
+++ b/testing/support.py
@@ -1,4 +1,4 @@
-import sys
+import sys, os
 
 if sys.version_info < (3,):
     __all__ = ['u']
@@ -86,3 +86,24 @@
         if not name.startswith('_') and not hasattr(module.ffi, name):
             setattr(ffi, name, NotImplemented)
     return module.lib
+
+
+# For testing, we call gcc with "-Werror".  This is fragile because newer
+# versions of gcc are always better at producing warnings, particularly for
+# auto-generated code.  We need here to adapt and silence them as needed.
+
+if sys.platform == 'win32':
+    extra_compile_args = []      # no obvious -Werror equivalent on MSVC
+else:
+    if (sys.platform == 'darwin' and
+          [int(x) for x in os.uname()[2].split('.')] >= [11, 0, 0]):
+        # assume a standard clang or gcc
+        extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion',
+                              '-Wno-unreachable-code']
+        # special things for clang
+        extra_compile_args.append('-Qunused-arguments')
+    else:
+        # assume a standard gcc
+        extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion',
+                              '-Wno-unused-parameter',
+                              '-Wno-unreachable-code']

From pypy.commits at gmail.com  Thu Nov  7 07:32:01 2019
From: pypy.commits at gmail.com (arigo)
Date: Thu, 07 Nov 2019 04:32:01 -0800 (PST)
Subject: [pypy-commit] pypy py3.6-exc-info: revert some parts, fix the rest
Message-ID: <5dc40ec1.1c69fb81.e4c99.8816@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.6-exc-info
Changeset: r97974:2857fcf6b913
Date: 2019-11-07 13:30 +0100
http://bitbucket.org/pypy/pypy/changeset/2857fcf6b913/

Log:	revert some parts, fix the rest

diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -32,9 +32,9 @@
         self.topframeref = jit.vref_None
         # this is exposed to app-level as 'sys.exc_info()'.  At any point in
         # time it is the exception caught by the topmost 'except ... as e:'
-        # app-level block.  (This is the *last* item in the list; previous
-        # items form a stack of older operrors.)
-        self.sys_exc_operrors = [None]
+        # app-level block.
+        self.sys_exc_operror = None
+        self.previous_operror_stack = []
         self.w_tracefunc = None
         self.is_tracing = 0
         self.compiler = space.createcompiler()
@@ -243,17 +243,23 @@
     def sys_exc_info(self):
         """Implements sys.exc_info().
         Return an OperationError instance or None.
+        Returns the "top-most" exception in the stack.
 
         # NOTE: the result is not the wrapped sys.exc_info() !!!
 
         """
-        i = len(self.sys_exc_operror) - 1
-        while i > 0 and self.sys_exc_operrors[i] is None:
-            i -= 1
-        return self.sys_exc_operrors[i]
+        result = self.sys_exc_operror
+        if result is None:
+            i = len(self.previous_operror_stack) - 1
+            while i >= 0:
+                result = self.previous_operror_stack[i]
+                if result is not None:
+                    break
+                i -= 1
+        return result
 
     def set_sys_exc_info(self, operror):
-        self.sys_exc_operrors[-1] = operror
+        self.sys_exc_operror = operror
 
     def set_sys_exc_info3(self, w_type, w_value, w_traceback):
         space = self.space
@@ -269,6 +275,21 @@
             operror = OperationError(w_type, w_value, tb)
         self.set_sys_exc_info(operror)
 
+    def enter_error_stack_item(self, saved_operr):
+        self.previous_operror_stack.append(saved_operr)
+
+    def leave_error_stack_item(self):
+        return self.previous_operror_stack.pop()
+
+    def fetch_and_clear_error_stack_state(self):
+        result = self.sys_exc_operror, self.previous_operror_stack
+        self.sys_exc_operror = None
+        self.previous_operror_stack = []
+        return result
+
+    def restore_error_stack_state(self, saved):
+        self.sys_exc_operror, self.previous_operror_stack = saved
+
     @jit.dont_look_inside
     def settrace(self, w_func):
         """Set the global trace function."""
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -105,11 +105,6 @@
         frame = self.frame
         if self.running:
             raise oefmt(space.w_ValueError, "%s already executing", self.KIND)
-        ec = space.getexecutioncontext()
-        current_exc_info = ec.sys_exc_info()
-        if self.saved_operr is not None:
-            ec.set_sys_exc_info(self.saved_operr)
-            self.saved_operr = None
         #
         # Optimization only: after we've started a Coroutine without
         # CO_YIELD_INSIDE_TRY, then Coroutine._finalize_() will be a no-op
@@ -123,6 +118,8 @@
                 raise oefmt(space.w_TypeError,
                             "can't send non-None value to a just-started %s",
                             self.KIND)
+        ec = space.getexecutioncontext()
+        ec.enter_error_stack_item(self.saved_operr)
         self.running = True
         try:
             w_result = frame.execute_frame(w_arg_or_err)
@@ -142,10 +139,8 @@
             self.running = False
             # note: this is not perfectly correct: see
             # test_exc_info_in_generator_4.  But it's simpler and
-            # bug-to-bug compatible with CPython 3.5.
-            if frame._any_except_or_finally_handler():
-                self.saved_operr = ec.sys_exc_info()
-            ec.set_sys_exc_info(current_exc_info)
+            # bug-to-bug compatible with CPython 3.5 and 3.6.
+            self.saved_operr = ec.leave_error_stack_item()
         return w_result
 
     def get_delegate(self):
diff --git a/pypy/module/__pypy__/test/test_magic.py b/pypy/module/__pypy__/test/test_magic.py
--- a/pypy/module/__pypy__/test/test_magic.py
+++ b/pypy/module/__pypy__/test/test_magic.py
@@ -60,8 +60,7 @@
         try:
             raise ValueError
         except ValueError as e:
-            pass
-        assert e.__context__ is terr
+            assert e.__context__ is terr
 
     def test_set_exc_info_issue3096(self):
         from __pypy__ import set_exc_info
diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py
--- a/pypy/module/_continuation/interp_continuation.py
+++ b/pypy/module/_continuation/interp_continuation.py
@@ -46,9 +46,9 @@
         #
         global_state.origin = self
         self.sthread = sthread
-        saved_exception = pre_switch(sthread)
+        saved_error_state = pre_switch(sthread)
         h = sthread.new(new_stacklet_callback)
-        post_switch(sthread, h, saved_exception)
+        post_switch(sthread, h, saved_error_state)
 
     def switch(self, w_to):
         sthread = self.sthread
@@ -84,9 +84,9 @@
             # double switch: the final destination is to.h
             global_state.destination = to
         #
-        saved_exception = pre_switch(sthread)
+        saved_error_state = pre_switch(sthread)
         h = sthread.switch(global_state.destination.h)
-        return post_switch(sthread, h, saved_exception)
+        return post_switch(sthread, h, saved_error_state)
 
     @unwrap_spec(w_value = WrappedDefault(None),
                  w_to = WrappedDefault(None))
@@ -257,11 +257,9 @@
     return self.h
 
 def pre_switch(sthread):
-    saved_exception = sthread.ec.sys_exc_info()
-    sthread.ec.set_sys_exc_info(None)
-    return saved_exception
+    return sthread.ec.fetch_and_clear_error_stack_state()
 
-def post_switch(sthread, h, saved_exception):
+def post_switch(sthread, h, saved_error_state):
     origin = global_state.origin
     self = global_state.destination
     global_state.origin = None
@@ -270,7 +268,7 @@
     #
     current = sthread.ec.topframeref
     sthread.ec.topframeref = self.bottomframe.f_backref
-    sthread.ec.set_sys_exc_info(saved_exception)
+    sthread.ec.restore_error_stack_state(saved_error_state)
     self.bottomframe.f_backref = origin.bottomframe.f_backref
     origin.bottomframe.f_backref = current
     #
diff --git a/pypy/module/sys/moduledef.py b/pypy/module/sys/moduledef.py
--- a/pypy/module/sys/moduledef.py
+++ b/pypy/module/sys/moduledef.py
@@ -209,31 +209,6 @@
         w_modules = self.get('modules')
         self.space.setitem(w_modules, w_name, w_module)
 
-    def getdictvalue(self, space, attr):
-        """ specialize access to dynamic exc_* attributes. """
-        value = MixedModule.getdictvalue(self, space, attr)
-        if value is not None:
-            return value
-        if attr == 'exc_type':
-            operror = space.getexecutioncontext().sys_exc_info()
-            if operror is None:
-                return space.w_None
-            else:
-                return operror.w_type
-        elif attr == 'exc_value':
-            operror = space.getexecutioncontext().sys_exc_info()
-            if operror is None:
-                return space.w_None
-            else:
-                return operror.get_w_value(space)
-        elif attr == 'exc_traceback':
-            operror = space.getexecutioncontext().sys_exc_info()
-            if operror is None:
-                return space.w_None
-            else:
-                return operror.get_w_traceback(space)
-        return None
-
     def get_flag(self, name):
         space = self.space
         return space.int_w(space.getattr(self.get('flags'), space.newtext(name)))
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -187,13 +187,6 @@
     else:
         return exc_info_without_tb(space, operror)
 
-def exc_clear(space):
-    """Clear global information on the current exception.  Subsequent calls
-to exc_info() will return (None,None,None) until another exception is
-raised and caught in the current thread or the execution stack returns to a
-frame where another exception is being handled."""
-    space.getexecutioncontext().clear_sys_exc_info()
-
 def settrace(space, w_func):
     """Set the global debug tracing function.  It will be called on each
 function call.  See the debugger chapter in the library manual."""

From pypy.commits at gmail.com  Thu Nov  7 07:36:41 2019
From: pypy.commits at gmail.com (arigo)
Date: Thu, 07 Nov 2019 04:36:41 -0800 (PST)
Subject: [pypy-commit] pypy py3.7: Merged in py3.7-bpo-29962 (pull request
 #681)
Message-ID: <5dc40fd9.1c69fb81.de05d.a9ba@mx.google.com>

Author: Armin Rigo <armin.rigo at gmail.com>
Branch: py3.7
Changeset: r97975:63e99c51c669
Date: 2019-11-07 12:36 +0000
http://bitbucket.org/pypy/pypy/changeset/63e99c51c669/

Log:	Merged in py3.7-bpo-29962 (pull request #681)

	bpo-29962: Implemented app-level math.remainder

diff --git a/pypy/module/math/app_math.py b/pypy/module/math/app_math.py
--- a/pypy/module/math/app_math.py
+++ b/pypy/module/math/app_math.py
@@ -47,3 +47,85 @@
     res, _, shift = _fac1(x)
     return res << shift
 
+
+def remainder(x, y):
+    """Difference between x and the closest integer multiple of y.
+
+    Return x - n*y where n*y is the closest integer multiple of y.
+    In the case where x is exactly halfway between two multiples of
+    y, the nearest even value of n is used. The result is always exact."""
+
+    from math import copysign, fabs, fmod, isfinite, isinf, isnan, nan
+
+    x = float(x)
+    y = float(y)
+
+    # Deal with most common case first.
+    if isfinite(x) and isfinite(y):
+        if y == 0.0:
+            # return nan
+            # Merging the logic from math_2 in CPython's mathmodule.c
+            # nan returned and x and y both not nan -> domain error
+            raise ValueError("math domain error")
+        
+        absx = fabs(x)
+        absy = fabs(y)
+        m = fmod(absx, absy)
+
+        # Warning: some subtlety here. What we *want* to know at this point is
+        # whether the remainder m is less than, equal to, or greater than half
+        # of absy. However, we can't do that comparison directly because we
+        # can't be sure that 0.5*absy is representable (the mutiplication
+        # might incur precision loss due to underflow). So instead we compare
+        # m with the complement c = absy - m: m < 0.5*absy if and only if m <
+        # c, and so on. The catch is that absy - m might also not be
+        # representable, but it turns out that it doesn't matter:
+        # - if m > 0.5*absy then absy - m is exactly representable, by
+        #     Sterbenz's lemma, so m > c
+        # - if m == 0.5*absy then again absy - m is exactly representable
+        #     and m == c
+        # - if m < 0.5*absy then either (i) 0.5*absy is exactly representable,
+        #     in which case 0.5*absy < absy - m, so 0.5*absy <= c and hence m <
+        #     c, or (ii) absy is tiny, either subnormal or in the lowest normal
+        #     binade. Then absy - m is exactly representable and again m < c.
+
+        c = absy - m
+        if m < c:
+            r = m
+        elif m > c:
+            r = -c
+        else:
+            # Here absx is exactly halfway between two multiples of absy,
+            # and we need to choose the even multiple. x now has the form
+            #     absx = n * absy + m
+            # for some integer n (recalling that m = 0.5*absy at this point).
+            # If n is even we want to return m; if n is odd, we need to
+            # return -m.
+            # So
+            #     0.5 * (absx - m) = (n/2) * absy
+            # and now reducing modulo absy gives us:
+            #                                     | m, if n is odd
+            #     fmod(0.5 * (absx - m), absy) = |
+            #                                     | 0, if n is even
+            # Now m - 2.0 * fmod(...) gives the desired result: m
+            # if n is even, -m if m is odd.
+            # Note that all steps in fmod(0.5 * (absx - m), absy)
+            # will be computed exactly, with no rounding error
+            # introduced.
+            assert m == c
+            r = m - 2.0 * fmod(0.5 * (absx - m), absy)
+        return copysign(1.0, x) * r
+    
+    # Special values.
+    if isnan(x):
+        return x
+    if isnan(y):
+        return y
+    if isinf(x):
+        # return nan
+        # Merging the logic from math_2 in CPython's mathmodule.c
+        # nan returned and x and y both not nan -> domain error
+        raise ValueError("math domain error")
+    assert isinf(y)
+    return x
+ 
\ No newline at end of file
diff --git a/pypy/module/math/moduledef.py b/pypy/module/math/moduledef.py
--- a/pypy/module/math/moduledef.py
+++ b/pypy/module/math/moduledef.py
@@ -5,6 +5,7 @@
 class Module(MixedModule):
     appleveldefs = {
        'factorial' : 'app_math.factorial',
+       'remainder' : 'app_math.remainder',
     }
 
     interpleveldefs = {
diff --git a/pypy/module/math/test/test_math.py b/pypy/module/math/test/test_math.py
--- a/pypy/module/math/test/test_math.py
+++ b/pypy/module/math/test/test_math.py
@@ -386,3 +386,17 @@
     def test_pi_tau(self):
         import math
         assert math.tau == math.pi * 2.0
+
+    def test_remainder(self):
+        import math
+        assert math.remainder(3, math.pi) == 3 - math.pi
+        assert math.remainder(-3, math.pi) == math.pi - 3
+        assert math.remainder(3, -math.pi) == 3 - math.pi
+        assert math.remainder(4, math.pi) == 4 - math.pi
+        assert math.remainder(6, math.pi) == 6 - 2 * math.pi
+        assert math.remainder(3, math.inf) == 3
+        assert math.remainder(3, -math.inf) == 3
+        assert math.isnan(math.remainder(3, math.nan))
+        assert math.isnan(math.remainder(math.nan, 3))
+        raises(ValueError, math.remainder, 3, 0)
+        raises(ValueError, math.remainder, math.inf, 3)

From pypy.commits at gmail.com  Thu Nov  7 07:56:12 2019
From: pypy.commits at gmail.com (arigo)
Date: Thu, 07 Nov 2019 04:56:12 -0800 (PST)
Subject: [pypy-commit] pypy py3.6-exc-info: oops, fix
Message-ID: <5dc4146c.1c69fb81.f357e.f208@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.6-exc-info
Changeset: r97976:7e2152a405d6
Date: 2019-11-07 13:54 +0100
http://bitbucket.org/pypy/pypy/changeset/7e2152a405d6/

Log:	oops, fix

diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -276,10 +276,15 @@
         self.set_sys_exc_info(operror)
 
     def enter_error_stack_item(self, saved_operr):
-        self.previous_operror_stack.append(saved_operr)
+        # 'sys_exc_operror' should be logically considered as the last
+        # item on the stack, so pushing a new item has the following effect:
+        self.previous_operror_stack.append(self.sys_exc_operror)
+        self.sys_exc_operror = saved_operr
 
     def leave_error_stack_item(self):
-        return self.previous_operror_stack.pop()
+        result = self.sys_exc_operror
+        self.sys_exc_operror = self.previous_operror_stack.pop()
+        return result
 
     def fetch_and_clear_error_stack_state(self):
         result = self.sys_exc_operror, self.previous_operror_stack

From pypy.commits at gmail.com  Thu Nov  7 08:08:13 2019
From: pypy.commits at gmail.com (arigo)
Date: Thu, 07 Nov 2019 05:08:13 -0800 (PST)
Subject: [pypy-commit] pypy py3.6-exc-info: close branch, ready to merge
Message-ID: <5dc4173d.1c69fb81.6fae9.a1f6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.6-exc-info
Changeset: r97977:a5983abcf08b
Date: 2019-11-07 13:56 +0100
http://bitbucket.org/pypy/pypy/changeset/a5983abcf08b/

Log:	close branch, ready to merge


From pypy.commits at gmail.com  Thu Nov  7 08:08:15 2019
From: pypy.commits at gmail.com (arigo)
Date: Thu, 07 Nov 2019 05:08:15 -0800 (PST)
Subject: [pypy-commit] pypy py3.6: hg merge py3.6-exc-info
Message-ID: <5dc4173f.1c69fb81.a454.b651@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.6
Changeset: r97978:251b47698e8e
Date: 2019-11-07 14:04 +0100
http://bitbucket.org/pypy/pypy/changeset/251b47698e8e/

Log:	hg merge py3.6-exc-info

	Generators need to store the old current 'exc_info' in a place that
	is visible, because in one corner case a call to sys.exc_info()
	might need it. See issue #3096

diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -34,6 +34,7 @@
         # time it is the exception caught by the topmost 'except ... as e:'
         # app-level block.
         self.sys_exc_operror = None
+        self.previous_operror_stack = []
         self.w_tracefunc = None
         self.is_tracing = 0
         self.compiler = space.createcompiler()
@@ -238,18 +239,62 @@
             self._trace(frame, 'exception', None, operationerr)
         #operationerr.print_detailed_traceback(self.space)
 
+    @jit.unroll_safe
     def sys_exc_info(self):
         """Implements sys.exc_info().
         Return an OperationError instance or None.
+        Returns the "top-most" exception in the stack.
 
         # NOTE: the result is not the wrapped sys.exc_info() !!!
 
         """
-        return self.sys_exc_operror
+        result = self.sys_exc_operror
+        if result is None:
+            i = len(self.previous_operror_stack) - 1
+            while i >= 0:
+                result = self.previous_operror_stack[i]
+                if result is not None:
+                    break
+                i -= 1
+        return result
 
     def set_sys_exc_info(self, operror):
         self.sys_exc_operror = operror
 
+    def set_sys_exc_info3(self, w_type, w_value, w_traceback):
+        space = self.space
+        if space.is_none(w_value):
+            operror = None
+        else:
+            tb = None
+            if not space.is_none(w_traceback):
+                try:
+                    tb = pytraceback.check_traceback(space, w_traceback, '?')
+                except OperationError:    # catch and ignore bogus objects
+                    pass
+            operror = OperationError(w_type, w_value, tb)
+        self.set_sys_exc_info(operror)
+
+    def enter_error_stack_item(self, saved_operr):
+        # 'sys_exc_operror' should be logically considered as the last
+        # item on the stack, so pushing a new item has the following effect:
+        self.previous_operror_stack.append(self.sys_exc_operror)
+        self.sys_exc_operror = saved_operr
+
+    def leave_error_stack_item(self):
+        result = self.sys_exc_operror
+        self.sys_exc_operror = self.previous_operror_stack.pop()
+        return result
+
+    def fetch_and_clear_error_stack_state(self):
+        result = self.sys_exc_operror, self.previous_operror_stack
+        self.sys_exc_operror = None
+        self.previous_operror_stack = []
+        return result
+
+    def restore_error_stack_state(self, saved):
+        self.sys_exc_operror, self.previous_operror_stack = saved
+
     @jit.dont_look_inside
     def settrace(self, w_func):
         """Set the global trace function."""
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -105,11 +105,6 @@
         frame = self.frame
         if self.running:
             raise oefmt(space.w_ValueError, "%s already executing", self.KIND)
-        ec = space.getexecutioncontext()
-        current_exc_info = ec.sys_exc_info()
-        if self.saved_operr is not None:
-            ec.set_sys_exc_info(self.saved_operr)
-            self.saved_operr = None
         #
         # Optimization only: after we've started a Coroutine without
         # CO_YIELD_INSIDE_TRY, then Coroutine._finalize_() will be a no-op
@@ -123,6 +118,8 @@
                 raise oefmt(space.w_TypeError,
                             "can't send non-None value to a just-started %s",
                             self.KIND)
+        ec = space.getexecutioncontext()
+        ec.enter_error_stack_item(self.saved_operr)
         self.running = True
         try:
             w_result = frame.execute_frame(w_arg_or_err)
@@ -142,10 +139,8 @@
             self.running = False
             # note: this is not perfectly correct: see
             # test_exc_info_in_generator_4.  But it's simpler and
-            # bug-to-bug compatible with CPython 3.5.
-            if frame._any_except_or_finally_handler():
-                self.saved_operr = ec.sys_exc_info()
-            ec.set_sys_exc_info(current_exc_info)
+            # bug-to-bug compatible with CPython 3.5 and 3.6.
+            self.saved_operr = ec.leave_error_stack_item()
         return w_result
 
     def get_delegate(self):
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -255,29 +255,7 @@
         if self._is_generator_or_coroutine():
             return self.initialize_as_generator(name, qualname)
         else:
-            # untranslated: check that sys_exc_info is exactly
-            # restored after running any Python function.
-            # Translated: actually save and restore it, as an attempt to
-            # work around rare cases that can occur if RecursionError or
-            # MemoryError is raised at just the wrong place
-            executioncontext = self.space.getexecutioncontext()
-            exc_on_enter = executioncontext.sys_exc_info()
-            if we_are_translated():
-                try:
-                    return self.execute_frame()
-                finally:
-                    executioncontext.set_sys_exc_info(exc_on_enter)
-            else:
-                # untranslated, we check consistency, but not in case of
-                # interp-level exceptions different than OperationError
-                # (e.g. a random failing test, or a pytest Skipped exc.)
-                try:
-                    w_res = self.execute_frame()
-                    assert exc_on_enter is executioncontext.sys_exc_info()
-                except OperationError:
-                    assert exc_on_enter is executioncontext.sys_exc_info()
-                    raise
-                return w_res
+            return self.execute_frame()
     run._always_inline_ = True
 
     def initialize_as_generator(self, name, qualname):
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -211,3 +211,7 @@
         return      # cpyext not imported yet, ignore
     from pypy.module.cpyext.api import invoke_pyos_inputhook
     invoke_pyos_inputhook(space)
+
+def set_exc_info(space, w_type, w_value, w_traceback=None):
+    ec = space.getexecutioncontext()
+    ec.set_sys_exc_info3(w_type, w_value, w_traceback)
diff --git a/pypy/module/__pypy__/moduledef.py b/pypy/module/__pypy__/moduledef.py
--- a/pypy/module/__pypy__/moduledef.py
+++ b/pypy/module/__pypy__/moduledef.py
@@ -118,6 +118,7 @@
         'fsdecode'                  : 'interp_magic.fsdecode',
         'pyos_inputhook'            : 'interp_magic.pyos_inputhook',
         'newmemoryview'             : 'interp_buffer.newmemoryview',
+        'set_exc_info'              : 'interp_magic.set_exc_info',
     }
 
     submodules = {
diff --git a/pypy/module/__pypy__/test/test_magic.py b/pypy/module/__pypy__/test/test_magic.py
--- a/pypy/module/__pypy__/test/test_magic.py
+++ b/pypy/module/__pypy__/test/test_magic.py
@@ -52,3 +52,28 @@
             pass
         a = A()
         assert _promote(a) is a
+
+    def test_set_exc_info(self):
+        from __pypy__ import set_exc_info
+        terr = TypeError("hello world")
+        set_exc_info(TypeError, terr)
+        try:
+            raise ValueError
+        except ValueError as e:
+            assert e.__context__ is terr
+
+    def test_set_exc_info_issue3096(self):
+        from __pypy__ import set_exc_info
+        def recover():
+            set_exc_info(None, None)
+        def main():
+            try:
+                raise RuntimeError('aaa')
+            finally:
+                recover()
+                raise RuntimeError('bbb')
+        try:
+            main()
+        except RuntimeError as e:
+            assert e.__cause__ is None
+            assert e.__context__ is None
diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py
--- a/pypy/module/_continuation/interp_continuation.py
+++ b/pypy/module/_continuation/interp_continuation.py
@@ -46,9 +46,9 @@
         #
         global_state.origin = self
         self.sthread = sthread
-        saved_exception = pre_switch(sthread)
+        saved_error_state = pre_switch(sthread)
         h = sthread.new(new_stacklet_callback)
-        post_switch(sthread, h, saved_exception)
+        post_switch(sthread, h, saved_error_state)
 
     def switch(self, w_to):
         sthread = self.sthread
@@ -84,9 +84,9 @@
             # double switch: the final destination is to.h
             global_state.destination = to
         #
-        saved_exception = pre_switch(sthread)
+        saved_error_state = pre_switch(sthread)
         h = sthread.switch(global_state.destination.h)
-        return post_switch(sthread, h, saved_exception)
+        return post_switch(sthread, h, saved_error_state)
 
     @unwrap_spec(w_value = WrappedDefault(None),
                  w_to = WrappedDefault(None))
@@ -257,11 +257,9 @@
     return self.h
 
 def pre_switch(sthread):
-    saved_exception = sthread.ec.sys_exc_info()
-    sthread.ec.set_sys_exc_info(None)
-    return saved_exception
+    return sthread.ec.fetch_and_clear_error_stack_state()
 
-def post_switch(sthread, h, saved_exception):
+def post_switch(sthread, h, saved_error_state):
     origin = global_state.origin
     self = global_state.destination
     global_state.origin = None
@@ -270,7 +268,7 @@
     #
     current = sthread.ec.topframeref
     sthread.ec.topframeref = self.bottomframe.f_backref
-    sthread.ec.set_sys_exc_info(saved_exception)
+    sthread.ec.restore_error_stack_state(saved_error_state)
     self.bottomframe.f_backref = origin.bottomframe.f_backref
     origin.bottomframe.f_backref = current
     #
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -403,8 +403,7 @@
 
 @cpython_api([PyObjectP, PyObjectP, PyObjectP], lltype.Void)
 def PyErr_GetExcInfo(space, ptype, pvalue, ptraceback):
-    """---Cython extension---
-
+    """
     Retrieve the exception info, as known from ``sys.exc_info()``.  This
     refers to an exception that was already caught, not to an exception
     that was freshly raised.  Returns new references for the three
@@ -432,8 +431,7 @@
 
 @cpython_api([PyObject, PyObject, PyObject], lltype.Void)
 def PyErr_SetExcInfo(space, py_type, py_value, py_traceback):
-    """---Cython extension---
-
+    """
     Set the exception info, as known from ``sys.exc_info()``.  This refers
     to an exception that was already caught, not to an exception that was
     freshly raised.  This function steals the references of the arguments.
@@ -450,19 +448,9 @@
     w_type = get_w_obj_and_decref(space, py_type)
     w_value = get_w_obj_and_decref(space, py_value)
     w_traceback = get_w_obj_and_decref(space, py_traceback)
-    if w_value is None or space.is_w(w_value, space.w_None):
-        operror = None
-    else:
-        tb = None
-        if w_traceback is not None:
-            try:
-                tb = pytraceback.check_traceback(space, w_traceback, '?')
-            except OperationError:    # catch and ignore bogus objects
-                pass
-        operror = OperationError(w_type, w_value, tb)
     #
     ec = space.getexecutioncontext()
-    ec.set_sys_exc_info(operror)
+    ec.set_sys_exc_info3(w_type, w_value, w_traceback)
 
 @cpython_api([], rffi.INT_real, error=CANNOT_FAIL)
 def PyOS_InterruptOccurred(space):
diff --git a/pypy/module/sys/moduledef.py b/pypy/module/sys/moduledef.py
--- a/pypy/module/sys/moduledef.py
+++ b/pypy/module/sys/moduledef.py
@@ -209,31 +209,6 @@
         w_modules = self.get('modules')
         self.space.setitem(w_modules, w_name, w_module)
 
-    def getdictvalue(self, space, attr):
-        """ specialize access to dynamic exc_* attributes. """
-        value = MixedModule.getdictvalue(self, space, attr)
-        if value is not None:
-            return value
-        if attr == 'exc_type':
-            operror = space.getexecutioncontext().sys_exc_info()
-            if operror is None:
-                return space.w_None
-            else:
-                return operror.w_type
-        elif attr == 'exc_value':
-            operror = space.getexecutioncontext().sys_exc_info()
-            if operror is None:
-                return space.w_None
-            else:
-                return operror.get_w_value(space)
-        elif attr == 'exc_traceback':
-            operror = space.getexecutioncontext().sys_exc_info()
-            if operror is None:
-                return space.w_None
-            else:
-                return operror.get_w_traceback(space)
-        return None
-
     def get_flag(self, name):
         space = self.space
         return space.int_w(space.getattr(self.get('flags'), space.newtext(name)))
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -187,13 +187,6 @@
     else:
         return exc_info_without_tb(space, operror)
 
-def exc_clear(space):
-    """Clear global information on the current exception.  Subsequent calls
-to exc_info() will return (None,None,None) until another exception is
-raised and caught in the current thread or the execution stack returns to a
-frame where another exception is being handled."""
-    space.getexecutioncontext().clear_sys_exc_info()
-
 def settrace(space, w_func):
     """Set the global debug tracing function.  It will be called on each
 function call.  See the debugger chapter in the library manual."""

From pypy.commits at gmail.com  Thu Nov  7 08:08:17 2019
From: pypy.commits at gmail.com (arigo)
Date: Thu, 07 Nov 2019 05:08:17 -0800 (PST)
Subject: [pypy-commit] pypy py3.6: merge heads
Message-ID: <5dc41741.1c69fb81.2591f.3efa@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.6
Changeset: r97979:2a48a020ef28
Date: 2019-11-07 14:07 +0100
http://bitbucket.org/pypy/pypy/changeset/2a48a020ef28/

Log:	merge heads

diff --git a/lib-python/3/test/test_dis.py b/lib-python/3/test/test_dis.py
--- a/lib-python/3/test/test_dis.py
+++ b/lib-python/3/test/test_dis.py
@@ -262,7 +262,7 @@
              20 RETURN_VALUE
 """
 
-# XXX: change for PyPy?
+# changed for PyPy
 dis_traceback = """\
 %3d           0 SETUP_EXCEPT            12 (to 14)
 
@@ -280,18 +280,18 @@
              22 POP_TOP
              24 STORE_FAST               0 (e)
              26 POP_TOP
-             28 SETUP_FINALLY           12 (to 42)
+             28 SETUP_FINALLY           10 (to 40)
 
 %3d          30 LOAD_FAST                0 (e)
              32 LOAD_ATTR                1 (__traceback__)
              34 STORE_FAST               1 (tb)
              36 POP_BLOCK
-             38 POP_EXCEPT
-             40 LOAD_CONST               0 (None)
-        >>   42 LOAD_CONST               0 (None)
-             44 STORE_FAST               0 (e)
-             46 DELETE_FAST              0 (e)
-             48 END_FINALLY
+             38 LOAD_CONST               0 (None)
+        >>   40 LOAD_CONST               0 (None)
+             42 STORE_FAST               0 (e)
+             44 DELETE_FAST              0 (e)
+             46 END_FINALLY
+             48 POP_EXCEPT
              50 JUMP_FORWARD             2 (to 54)
         >>   52 END_FINALLY
 
@@ -686,9 +686,9 @@
 # End fodder for opinfo generation tests
 expected_outer_line = 1
 _line_offset = outer.__code__.co_firstlineno - 1
-code_object_f = outer.__code__.co_consts[3]
+code_object_f = outer.__code__.co_consts[2]
 expected_f_line = code_object_f.co_firstlineno - _line_offset
-code_object_inner = code_object_f.co_consts[3]
+code_object_inner = code_object_f.co_consts[2]
 expected_inner_line = code_object_inner.co_firstlineno - _line_offset
 expected_jumpy_line = 1
 
@@ -713,22 +713,22 @@
 
 Instruction = dis.Instruction
 expected_opinfo_outer = [
-  Instruction(opname='LOAD_CONST', opcode=100, arg=8, argval=(3, 4), argrepr='(3, 4)', offset=0, starts_line=2, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=1, argval=(3, 4), argrepr='(3, 4)', offset=0, starts_line=2, is_jump_target=False),
   Instruction(opname='LOAD_CLOSURE', opcode=135, arg=0, argval='a', argrepr='a', offset=2, starts_line=None, is_jump_target=False),
   Instruction(opname='LOAD_CLOSURE', opcode=135, arg=1, argval='b', argrepr='b', offset=4, starts_line=None, is_jump_target=False),
   Instruction(opname='BUILD_TUPLE', opcode=102, arg=2, argval=2, argrepr='', offset=6, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=code_object_f, argrepr=repr(code_object_f), offset=8, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='outer.<locals>.f', argrepr="'outer.<locals>.f'", offset=10, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=code_object_f, argrepr=repr(code_object_f), offset=8, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval='outer.<locals>.f', argrepr="'outer.<locals>.f'", offset=10, starts_line=None, is_jump_target=False),
   Instruction(opname='MAKE_FUNCTION', opcode=132, arg=9, argval=9, argrepr='', offset=12, starts_line=None, is_jump_target=False),
   Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='f', argrepr='f', offset=14, starts_line=None, is_jump_target=False),
   Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=16, starts_line=7, is_jump_target=False),
   Instruction(opname='LOAD_DEREF', opcode=136, arg=0, argval='a', argrepr='a', offset=18, starts_line=None, is_jump_target=False),
   Instruction(opname='LOAD_DEREF', opcode=136, arg=1, argval='b', argrepr='b', offset=20, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval='', argrepr="''", offset=22, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval=1, argrepr='1', offset=24, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='', argrepr="''", offset=22, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=24, starts_line=None, is_jump_target=False),
   Instruction(opname='BUILD_LIST', opcode=103, arg=0, argval=0, argrepr='', offset=26, starts_line=None, is_jump_target=False),
   Instruction(opname='BUILD_MAP', opcode=105, arg=0, argval=0, argrepr='', offset=28, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=7, argval='Hello world!', argrepr="'Hello world!'", offset=30, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval='Hello world!', argrepr="'Hello world!'", offset=30, starts_line=None, is_jump_target=False),
   Instruction(opname='CALL_FUNCTION', opcode=131, arg=7, argval=7, argrepr='', offset=32, starts_line=None, is_jump_target=False),
   Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=34, starts_line=None, is_jump_target=False),
   Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='f', argrepr='f', offset=36, starts_line=8, is_jump_target=False),
@@ -736,14 +736,14 @@
 ]
 
 expected_opinfo_f = [
-  Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=(5, 6), argrepr='(5, 6)', offset=0, starts_line=3, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=1, argval=(5, 6), argrepr='(5, 6)', offset=0, starts_line=3, is_jump_target=False),
   Instruction(opname='LOAD_CLOSURE', opcode=135, arg=2, argval='a', argrepr='a', offset=2, starts_line=None, is_jump_target=False),
   Instruction(opname='LOAD_CLOSURE', opcode=135, arg=3, argval='b', argrepr='b', offset=4, starts_line=None, is_jump_target=False),
   Instruction(opname='LOAD_CLOSURE', opcode=135, arg=0, argval='c', argrepr='c', offset=6, starts_line=None, is_jump_target=False),
   Instruction(opname='LOAD_CLOSURE', opcode=135, arg=1, argval='d', argrepr='d', offset=8, starts_line=None, is_jump_target=False),
   Instruction(opname='BUILD_TUPLE', opcode=102, arg=4, argval=4, argrepr='', offset=10, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=code_object_inner, argrepr=repr(code_object_inner), offset=12, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='outer.<locals>.f.<locals>.inner', argrepr="'outer.<locals>.f.<locals>.inner'", offset=14, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=code_object_inner, argrepr=repr(code_object_inner), offset=12, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval='outer.<locals>.f.<locals>.inner', argrepr="'outer.<locals>.f.<locals>.inner'", offset=14, starts_line=None, is_jump_target=False),
   Instruction(opname='MAKE_FUNCTION', opcode=132, arg=9, argval=9, argrepr='', offset=16, starts_line=None, is_jump_target=False),
   Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='inner', argrepr='inner', offset=18, starts_line=None, is_jump_target=False),
   Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=20, starts_line=5, is_jump_target=False),
diff --git a/lib-python/3/test/test_extcall.py b/lib-python/3/test/test_extcall.py
--- a/lib-python/3/test/test_extcall.py
+++ b/lib-python/3/test/test_extcall.py
@@ -57,7 +57,7 @@
     Traceback (most recent call last):
         ...
     TypeError: ...got multiple values for keyword argument 'a'
-    >>> f(1, 2, a=3, **{'a': 4}, **{'a': 5})
+    >>> f(1, 2, a=3, **{'a': 4}, **{'a': 5})  #doctest: +ELLIPSIS
     Traceback (most recent call last):
         ...
     TypeError: ...got multiple values for keyword argument 'a'
@@ -254,20 +254,21 @@
       ...
     TypeError: h() argument after * must be an iterable, not function
 
-    >>> h(*[1], *h)
+    >>> h(*[1], *h)  #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: h() argument after * must be an iterable, not function
+    TypeError: ...
 
     >>> dir(*h)
     Traceback (most recent call last):
       ...
     TypeError: dir() argument after * must be an iterable, not function
 
-    >>> None(*h)
+    >>> None(**h)  #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: ...argument after * must be an iterable, not function
+    TypeError: ... object argument after ** must be a mapping, \
+not function
 
     >>> h(**h)
     Traceback (most recent call last):
@@ -289,35 +290,20 @@
       ...
     TypeError: h() argument after ** must be a mapping, not list
 
-    >>> h(**{'a': 1}, **h)
+    >>> h(**{'a': 1}, **h)  #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: h() argument after ** must be a mapping, not function
+    TypeError: ...argument after ** must be a mapping, not function
 
-    >>> h(**{'a': 1}, **[])
+    >>> h(**{'a': 1}, **[])  #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: h() argument after ** must be a mapping, not list
+    TypeError: ...argument after ** must be a mapping, not list
 
     >>> dir(**h)
     Traceback (most recent call last):
       ...
-    TypeError: ...argument after * must be an iterable, not function
-
-    >>> None(*h)                               #doctest: +ELLIPSIS
-    Traceback (most recent call last):
-      ...
-    TypeError: ...argument after * must be an iterable, not function
-
-    >>> h(**h)                                 #doctest: +ELLIPSIS
-    Traceback (most recent call last):
-      ...
-    TypeError: ...argument after ** must be a mapping, not function
-
-    >>> dir(**h)                               #doctest: +ELLIPSIS
-    Traceback (most recent call last):
-      ...
-    TypeError: ...argument after ** must be a mapping, not function
+    TypeError: dir() argument after ** must be a mapping, not function
 
     >>> None(**h)                              #doctest: +ELLIPSIS
     Traceback (most recent call last):
diff --git a/lib-python/3/test/test_flufl.py b/lib-python/3/test/test_flufl.py
--- a/lib-python/3/test/test_flufl.py
+++ b/lib-python/3/test/test_flufl.py
@@ -15,7 +15,7 @@
         self.assertEqual(cm.exception.text, '2 != 3\n')
         self.assertEqual(cm.exception.filename, '<FLUFL test>')
         self.assertEqual(cm.exception.lineno, 2)
-        self.assertEqual(cm.exception.offset, 4)
+        self.assertEqual(cm.exception.offset, 2)  # changed in PyPy
 
     def test_guido_as_bdfl(self):
         code = '2 {0} 3'
@@ -26,7 +26,7 @@
         self.assertEqual(cm.exception.text, '2 <> 3\n')
         self.assertEqual(cm.exception.filename, '<FLUFL test>')
         self.assertEqual(cm.exception.lineno, 1)
-        self.assertEqual(cm.exception.offset, 4)
+        self.assertEqual(cm.exception.offset, 2)  # changed in PyPy
 
 
 if __name__ == '__main__':
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -218,10 +218,6 @@
         BoolOption("newshortcut",
                    "cache and shortcut calling __new__ from builtin types",
                    default=False),
-        BoolOption("reinterpretasserts",
-                   "Perform reinterpretation when an assert fails "
-                   "(only relevant for tests)",
-                   default=False),
 
      ]),
 ])
diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -135,8 +135,8 @@
         return PyPyModule(path, parent)
 
 def is_applevel(item):
-    from pypy.tool.pytest.apptest import AppTestFunction
-    return isinstance(item, AppTestFunction)
+    from pypy.tool.pytest.apptest import AppTestMethod
+    return isinstance(item, AppTestMethod)
 
 def pytest_collection_modifyitems(config, items):
     if config.getoption('runappdirect') or config.getoption('direct_apptest'):
@@ -166,8 +166,6 @@
     def funcnamefilter(self, name):
         if name.startswith('test_'):
             return self.accept_regular_test()
-        if name.startswith('app_test_'):
-            return True
         return False
 
     def classnamefilter(self, name):
@@ -182,13 +180,6 @@
             if name.startswith('AppTest'):
                 from pypy.tool.pytest.apptest import AppClassCollector
                 return AppClassCollector(name, parent=self)
-
-        elif hasattr(obj, 'func_code') and self.funcnamefilter(name):
-            if name.startswith('app_test_'):
-                assert not obj.func_code.co_flags & 32, \
-                    "generator app level functions? you must be joking"
-                from pypy.tool.pytest.apptest import AppTestFunction
-                return AppTestFunction(name, parent=self)
         return super(PyPyModule, self).makeitem(name, obj)
 
 def skip_on_missing_buildoption(**ropts):
@@ -207,27 +198,15 @@
     py.test.skip("need translated pypy3 with: %s, got %s"
                  %(ropts,options))
 
-class LazyObjSpaceGetter(object):
-    def __get__(self, obj, cls=None):
-        from pypy.tool.pytest.objspace import gettestobjspace
-        space = gettestobjspace()
-        if cls:
-            cls.space = space
-        return space
-
-
 @pytest.hookimpl(tryfirst=True)
 def pytest_runtest_setup(item):
     if isinstance(item, py.test.collect.Function):
         appclass = item.getparent(py.test.Class)
         if appclass is not None:
+            from pypy.tool.pytest.objspace import gettestobjspace
             # Make cls.space and cls.runappdirect available in tests.
-            spaceconfig = getattr(appclass.obj, 'spaceconfig', None)
-            if spaceconfig is not None:
-                from pypy.tool.pytest.objspace import gettestobjspace
-                appclass.obj.space = gettestobjspace(**spaceconfig)
-            else:
-                appclass.obj.space = LazyObjSpaceGetter()
+            spaceconfig = getattr(appclass.obj, 'spaceconfig', {})
+            appclass.obj.space = gettestobjspace(**spaceconfig)
             appclass.obj.runappdirect = option.runappdirect
 
 def pytest_ignore_collect(path, config):
diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -497,10 +497,10 @@
 
     def handle_async_funcdef(self, node, decorators=None):
         return self.handle_funcdef_impl(node.get_child(1), 1, decorators)
-    
+
     def handle_funcdef(self, node, decorators=None):
         return self.handle_funcdef_impl(node, 0, decorators)
-    
+
     def handle_async_stmt(self, node):
         ch = node.get_child(1)
         if ch.type == syms.funcdef:
@@ -942,7 +942,7 @@
                 if flufl and comp_node.get_value() == '!=':
                     self.error("with Barry as BDFL, use '<>' instead of '!='", comp_node)
                 elif not flufl and comp_node.get_value() == '<>':
-                    self.error('invalid comparison', comp_node)
+                    self.error('invalid syntax', comp_node)
                 return ast.NotEq
             elif comp_type == tokens.NAME:
                 if comp_node.get_value() == "is":
@@ -1014,7 +1014,7 @@
                              atom_node.get_column())
         else:
             return atom_expr
-    
+
     def handle_power(self, power_node):
         atom_expr = self.handle_atom_expr(power_node.get_child(0))
         if power_node.num_children() == 1:
@@ -1092,7 +1092,7 @@
     def handle_call(self, args_node, callable_expr):
         arg_count = 0 # position args + iterable args unpackings
         keyword_count = 0 # keyword args + keyword args unpackings
-        generator_count = 0 
+        generator_count = 0
         for i in range(args_node.num_children()):
             argument = args_node.get_child(i)
             if argument.type == syms.argument:
@@ -1300,7 +1300,6 @@
                     if is_dict:
                         raise self.error("dict unpacking cannot be used in "
                                          "dict comprehension", atom_node)
-                    
                     return self.handle_dictcomp(maker, atom_node)
                 else:
                     # a dictionary display
@@ -1423,7 +1422,7 @@
         comps = self.comprehension_helper(dict_maker.get_child(i))
         return ast.DictComp(key, value, comps, atom_node.get_lineno(),
                                                atom_node.get_column())
-    
+
     def handle_dictdisplay(self, node, atom_node):
         keys = []
         values = []
@@ -1435,7 +1434,7 @@
             i += 1
         return ast.Dict(keys, values, atom_node.get_lineno(),
                                       atom_node.get_column())
-    
+
     def handle_setdisplay(self, node, atom_node):
         elts = []
         i = 0
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -393,6 +393,14 @@
             self.emit_op_arg(ops.BUILD_CONST_KEY_MAP, l)
         return l
 
+    def _visit_defaults(self, defaults):
+        w_tup = self._tuple_of_consts(defaults)
+        if w_tup:
+            self.load_const(w_tup)
+        else:
+            self.visit_sequence(defaults)
+            self.emit_op_arg(ops.BUILD_TUPLE, len(defaults))
+
     @specialize.arg(2)
     def _visit_function(self, func, function_code_generator):
         self.update_position(func.lineno, True)
@@ -403,11 +411,10 @@
         assert isinstance(args, ast.arguments)
 
         oparg = 0
-        self.visit_sequence(args.defaults)
 
         if args.defaults is not None and len(args.defaults):
             oparg = oparg | 0x01
-            self.emit_op_arg(ops.BUILD_TUPLE, len(args.defaults))
+            self._visit_defaults(args.defaults)
 
         if args.kwonlyargs:
             kw_default_count = self._visit_kwonlydefaults(args)
@@ -438,12 +445,10 @@
         args = lam.args
         assert isinstance(args, ast.arguments)
 
-        self.visit_sequence(args.defaults)
-
         oparg = 0
         if args.defaults is not None and len(args.defaults):
             oparg = oparg | 0x01
-            self.emit_op_arg(ops.BUILD_TUPLE, len(args.defaults))
+            self._visit_defaults(args.defaults)
 
         if args.kwonlyargs:
             kw_default_count = self._visit_kwonlydefaults(args)
diff --git a/pypy/interpreter/astcompiler/test/apptest_misc.py b/pypy/interpreter/astcompiler/test/apptest_misc.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/astcompiler/test/apptest_misc.py
@@ -0,0 +1,18 @@
+def test_warning_to_error_translation():
+    import warnings
+    statement = """\
+def wrong1():
+    a = 1
+    b = 2
+    global a
+    global b
+"""
+    with warnings.catch_warnings():
+        warnings.filterwarnings("error", module="<test string>")
+        try:
+            compile(statement, '<test string>', 'exec')
+        except SyntaxError as err:
+            assert err.lineno is not None
+            assert err.filename is not None
+            assert err.offset is not None
+            assert err.msg is not None
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1548,6 +1548,30 @@
         counts = self.count_instructions(source)
         assert ops.BUILD_TUPLE not in counts
 
+    def test_fold_defaults_tuple(self):
+        source = """def f():
+            def g(a, b=2, c=None, d='foo'):
+                return None
+            return g
+        """
+        counts = self.count_instructions(source)
+        assert ops.BUILD_TUPLE not in counts
+
+        source = """def f():
+            g = lambda a, b=2, c=None, d='foo': None
+            return g
+        """
+        counts = self.count_instructions(source)
+        assert ops.BUILD_TUPLE not in counts
+
+        source = """def f():
+            def g(a, b=2, c=None, d=[]):
+                return None
+            return g
+        """
+        counts = self.count_instructions(source)
+        assert counts[ops.BUILD_TUPLE] == 1
+
     def test_constant_tuples_star(self):
         source = """def f(a, c):
             return (u"a", 1, *a, 3, 5, 3, *c)
diff --git a/pypy/interpreter/astcompiler/test/test_misc.py b/pypy/interpreter/astcompiler/test/test_misc.py
--- a/pypy/interpreter/astcompiler/test/test_misc.py
+++ b/pypy/interpreter/astcompiler/test/test_misc.py
@@ -43,26 +43,6 @@
     Instruction(ops.LOAD_FAST, 0x5030201).encode(c)
     assert c == [chr(ops.EXTENDED_ARG), '\x05', chr(ops.EXTENDED_ARG), '\x03', chr(ops.EXTENDED_ARG), '\x02', chr(ops.LOAD_FAST), '\x01']
 
-def app_test_warning_to_error_translation():
-    import warnings
-
-    with warnings.catch_warnings():
-        warnings.filterwarnings("error", module="<test string>")
-        statement = """\
-def wrong1():
-    a = 1
-    b = 2
-    global a
-    global b
-"""
-        try:
-           compile(statement, '<test string>', 'exec')
-        except SyntaxError as err:
-           assert err.lineno is not None
-           assert err.filename is not None
-           assert err.offset is not None
-           assert err.msg is not None
-
 def test_encode_lnotab_pair():
     l = []
     _encode_lnotab_pair(0, 1, l)
diff --git a/pypy/interpreter/test/test_appinterp.py b/pypy/interpreter/test/test_appinterp.py
--- a/pypy/interpreter/test/test_appinterp.py
+++ b/pypy/interpreter/test/test_appinterp.py
@@ -98,10 +98,6 @@
     w_clsattr = space.getattr(c, space.wrap('attr'))
     assert space.eq_w(w_clsattr, space.wrap(17))
 
-def app_test_something_at_app_level():
-    x = 2
-    assert x/2 == 1
-
 class AppTestMethods:
     def test_some_app_test_method(self):
         assert 2 == 2
diff --git a/pypy/objspace/std/test/test_operation.py b/pypy/objspace/std/test/apptest_operation.py
rename from pypy/objspace/std/test/test_operation.py
rename to pypy/objspace/std/test/apptest_operation.py
--- a/pypy/objspace/std/test/test_operation.py
+++ b/pypy/objspace/std/test/apptest_operation.py
@@ -1,10 +1,11 @@
+from pytest import raises
 
+def teq(a, b):
+    assert a == b
+    assert type(a) is type(b)
 
-def app_test_int_vs_float():
-    def teq(a, b):
-        assert a == b
-        assert type(a) is type(b)
 
+def test_int_vs_float():
     # binary operators
     teq( 5  - 2    , 3   )
     teq( 5  - 2.0  , 3.0 )
diff --git a/pypy/tool/pytest/apptest.py b/pypy/tool/pytest/apptest.py
--- a/pypy/tool/pytest/apptest.py
+++ b/pypy/tool/pytest/apptest.py
@@ -1,7 +1,7 @@
 # Collects and executes application-level tests.
 #
-# Classes which names start with "AppTest", or function which names
-# start with "app_test*" are not executed by the host Python, but
+# Classes which names start with "AppTest"
+# are not executed by the host Python, but
 # by an interpreted pypy object space.
 #
 # ...unless the -A option ('runappdirect') is passed.
@@ -232,7 +232,7 @@
         return fn
 
 
-class AppTestFunction(py.test.collect.Function):
+class AppTestMethod(py.test.collect.Function):
     def _prunetraceback(self, traceback):
         return traceback
 
@@ -252,21 +252,10 @@
                 raise AppError, AppError(appexcinfo), tb
             raise
 
-    def runtest(self):
-        target = self.obj
-        src = extract_docstring_if_empty_function(target)
-        if self.config.option.runappdirect:
-            return run_with_python(self.config.option.python, src, None)
-        space = gettestobjspace(**{'objspace.std.reinterpretasserts': True})
-        filename = self._getdynfilename(target)
-        func = app2interp_temp(src, filename=filename)
-        # print "executing", func
-        self.execute_appex(space, func, space)
-
     def repr_failure(self, excinfo):
         if excinfo.errisinstance(AppError):
             excinfo = excinfo.value.excinfo
-        return super(AppTestFunction, self).repr_failure(excinfo)
+        return super(AppTestMethod, self).repr_failure(excinfo)
 
     def _getdynfilename(self, func):
         code = getattr(func, 'im_func', func).func_code
@@ -279,8 +268,6 @@
         if hasattr(self, 'space'):
             self.space.getexecutioncontext()._run_finalizers_now()
 
-
-class AppTestMethod(AppTestFunction):
     def setup(self):
         super(AppTestMethod, self).setup()
         instance = self.parent.obj
diff --git a/pypy/tool/pytest/objspace.py b/pypy/tool/pytest/objspace.py
--- a/pypy/tool/pytest/objspace.py
+++ b/pypy/tool/pytest/objspace.py
@@ -33,9 +33,6 @@
     config.objspace.extmodules = 'pypy.tool.pytest.fake_pytest'
     space = make_objspace(config)
     space.startup() # Initialize all builtin modules
-    if config.objspace.std.reinterpretasserts:
-        space.setitem(space.builtin.w_dict, space.wrap('AssertionError'),
-                    appsupport.build_pytest_assertion(space))
     space.setitem(space.builtin.w_dict, space.wrap('raises'),
                   space.wrap(appsupport.app_raises))
     space.setitem(space.builtin.w_dict, space.wrap('skip'),
diff --git a/pypy/tool/pytest/pypy_test_failure_demo.py b/pypy/tool/pytest/pypy_test_failure_demo.py
--- a/pypy/tool/pytest/pypy_test_failure_demo.py
+++ b/pypy/tool/pytest/pypy_test_failure_demo.py
@@ -1,39 +1,14 @@
-class AppTestTest: 
+class AppTestTest:
     def test_app_method(self):
-        assert 42 == 41 
+        assert 42 == 41
 
-def app_test_app_func(): 
-    assert 41 == 42 
-
-def test_interp_func(space): 
-    assert space.is_true(space.w_None) 
+def test_interp_func(space):
+    assert space.is_true(space.w_None)
 
 def test_interp_reinterpret(space):
     a = 1
     assert a == 2
 
-class TestInterpTest: 
-    def test_interp_method(self): 
-        assert self.space.is_true(self.space.w_False) 
-
-def app_test_raises_in_statement():
-    raises(ValueError, """
-        y = x  # name error
-    """)
-
-def app_test_raises_something():
-    int("hallo") 
-
-def app_test_raises_wrong1():
-    raises(SyntaxError, 'int("hello")')
-
-def app_test_raises_wrong2():
-    raises(SyntaxError, int, "hello") 
-
-def app_test_raises_doesnt():
-    raises(ValueError, int, 3)
-
-def app_test_skip():
-    skip("skipped test")
-    
-    
+class TestInterpTest:
+    def test_interp_method(self):
+        assert self.space.is_true(self.space.w_False)
diff --git a/pypy/tool/pytest/test/conftest1_innertest.py b/pypy/tool/pytest/test/conftest1_innertest.py
--- a/pypy/tool/pytest/test/conftest1_innertest.py
+++ b/pypy/tool/pytest/test/conftest1_innertest.py
@@ -1,17 +1,9 @@
 
-def test_something(space): 
-    assert space.w_None is space.w_None 
+def test_something(space):
+    assert space.w_None is space.w_None
 
-def app_test_something(): 
-    assert 42 == 42 
-
-def app_test_code_in_docstring_failing():
-    """
-    assert False
-    """
-
-class AppTestSomething: 
-    def test_method_app(self): 
+class AppTestSomething:
+    def test_method_app(self):
         assert 23 == 23
 
     def test_code_in_docstring_failing(self):
@@ -26,12 +18,7 @@
         """
         assert True
 
-    
+
 class TestSomething:
-    def test_method(self): 
-        assert self.space 
- 
-def app_test_raise_in_a_closure():
-    def f(x):
-        raises(AttributeError, "x.foo")
-    f(42)
+    def test_method(self):
+        assert self.space
diff --git a/pypy/tool/pytest/test/test_appsupport.py b/pypy/tool/pytest/test/test_appsupport.py
--- a/pypy/tool/pytest/test/test_appsupport.py
+++ b/pypy/tool/pytest/test/test_appsupport.py
@@ -77,8 +77,6 @@
 def test_applevel_raises_simple_display(testdir):
     setpypyconftest(testdir)
     p = testdir.makepyfile("""
-        def app_test_raises():
-            raises(ValueError, x)
         class AppTestRaises:
             def test_func(self):
                 raises (ValueError, x)
@@ -87,7 +85,6 @@
     result = testdir.runpytest(p, "-s")
     assert result.ret == 1
     result.stdout.fnmatch_lines([
-        "*E*application-level*NameError*x*not defined",
         "*test_func(self)*",
         ">*raises*ValueError*",
         "*E*application-level*NameError*x*not defined",
@@ -99,37 +96,6 @@
         "*E*application-level*NameError*x*not defined",
     ])
 
-def test_applevel_raises_display(testdir):
-    setpypyconftest(testdir)
-    p = testdir.makepyfile("""
-        def app_test_raises():
-            raises(ValueError, "x")
-            pass
-    """)
-    result = testdir.runpytest(p, "-s")
-    assert result.ret == 1
-    result.stdout.fnmatch_lines([
-        "*E*application-level*NameError*x*not defined",
-    ])
-    result = testdir.runpytest(p) # this time we may run the pyc file
-    assert result.ret == 1
-    result.stdout.fnmatch_lines([
-        "*E*application-level*NameError*x*not defined",
-    ])
-
-def test_applevel_raise_keyerror(testdir):
-    setpypyconftest(testdir)
-    p = testdir.makepyfile("""
-        def app_test_raises():
-            raise KeyError(42)
-            pass
-    """)
-    result = testdir.runpytest(p, "-s")
-    assert result.ret == 1
-    result.stdout.fnmatch_lines([
-        "*E*application-level*KeyError*42*",
-    ])
-
 def test_apptest_raise(testdir):
     setpypyconftest(testdir)
     p = testdir.makepyfile(apptest_raise="""
@@ -170,17 +136,6 @@
         "*E*+ bar*",
     ])
 
-
-def app_test_raises():
-    info = raises(TypeError, id)
-    assert info.type is TypeError
-    assert isinstance(info.value, TypeError)
-
-    x = 43
-    info = raises(ZeroDivisionError, "x/0")
-    assert info.type is ZeroDivisionError
-    assert isinstance(info.value, ZeroDivisionError)
-
 def test_rename_module():
     from pypy.tool.pytest.apptest import _rename_module
     assert _rename_module("sys") == "sys"
diff --git a/pypy/tool/pytest/test/test_conftest1.py b/pypy/tool/pytest/test/test_conftest1.py
--- a/pypy/tool/pytest/test/test_conftest1.py
+++ b/pypy/tool/pytest/test/test_conftest1.py
@@ -41,10 +41,9 @@
 
     def test_selection_by_keyword_app(self):
         passed, failed = subproc_run("-m", "applevel -docstring", innertest)
-        assert len(passed) == 4
-        assert len(failed) == 2
-        assert "app_test_something" in passed[0]
-        assert "test_method_app" in passed[1]
+        assert len(passed) == 2
+        assert len(failed) == 1
+        assert "test_method_app" in passed[0]
 
     def test_docstring_in_methods(self):
         passed, failed = subproc_run("-k", "AppTestSomething and test_code_in_docstring",
@@ -54,12 +53,6 @@
         assert "test_code_in_docstring_ignored" in passed[0]
         assert "test_code_in_docstring_failing" in failed[0]
 
-    def test_docstring_in_functions(self):
-        passed, failed = subproc_run("-k", "app_test_code_in_docstring", innertest)
-        assert passed == []
-        assert len(failed) == 1
-        assert "app_test_code_in_docstring_failing" in failed[0]
-
     @py.test.mark.xfail(reason='fails on buildslave')
     def test_docstring_runappdirect(self):
         passed, failed = subproc_run(innertest,
diff --git a/pypy/tool/pytest/test/test_pytestsupport.py b/pypy/tool/pytest/test/test_pytestsupport.py
--- a/pypy/tool/pytest/test/test_pytestsupport.py
+++ b/pypy/tool/pytest/test/test_pytestsupport.py
@@ -6,21 +6,6 @@
 
 pytest_plugins = "pytester"
 
-def app_test_exception():
-    try:
-        raise AssertionError("42")
-    except AssertionError:
-        pass
-    else:
-        raise AssertionError("app level AssertionError mixup!")
-
-def app_test_exception_with_message():
-    try:
-        assert 0, "Failed"
-    except AssertionError as e:
-        assert e.msg == "Failed"
-
-
 def test_appexecinfo(space):
     try:
         space.appexec([], "(): raise ValueError")

From pypy.commits at gmail.com  Thu Nov  7 11:44:56 2019
From: pypy.commits at gmail.com (Yannick_Jadoul)
Date: Thu, 07 Nov 2019 08:44:56 -0800 (PST)
Subject: [pypy-commit] pypy py3.7-pep564: Using r_int64 to handle
 nanoseconds in time module without overflowing
Message-ID: <5dc44a08.1c69fb81.832ee.c65a@mx.google.com>

Author: Yannick Jadoul <yannick.jadoul at belgacom.net>
Branch: py3.7-pep564
Changeset: r97980:b2be19ccfa88
Date: 2019-11-07 17:44 +0100
http://bitbucket.org/pypy/pypy/changeset/b2be19ccfa88/

Log:	Using r_int64 to handle nanoseconds in time module without
	overflowing

diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -10,7 +10,7 @@
     str_decode_locale_surrogateescape, unicode_encode_locale_surrogateescape)
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rlib.rarithmetic import (
-    intmask, r_ulonglong, r_longfloat, widen, ovfcheck, ovfcheck_float_to_int)
+    intmask, r_ulonglong, r_longfloat, r_int64, widen, ovfcheck, ovfcheck_float_to_int)
 from rpython.rlib.rtime import (GETTIMEOFDAY_NO_TZ, TIMEVAL,
                                 HAVE_GETTIMEOFDAY, HAVE_FTIME)
 from rpython.rlib import rposix, rtime
@@ -277,7 +277,7 @@
                     _setinfo(space, w_info, "GetSystemTimeAsFileTime()",
                              time_increment[0] * 1e-7, False, True)
             if return_ns:
-                return space.newint(microseconds * 10**3)
+                return space.newint(r_int64(microseconds) * 10**3)
             else:
                 tv_sec = microseconds / 10**6
                 tv_usec = microseconds % 10**6
@@ -303,8 +303,8 @@
                         _setinfo(space, w_info, "gettimeofday()", 1e-6, False, True)
                     if return_ns:
                         return space.newint(
-                            widen(timeval.c_tv_sec) * 10**9 +
-                            widen(timeval.c_tv_usec) * 10**3)
+                            r_int64(timeval.c_tv_sec) * 10**9 +
+                            r_int64(timeval.c_tv_usec) * 10**3)
                     else:
                         return space.newfloat(
                             widen(timeval.c_tv_sec) +
@@ -316,8 +316,8 @@
                     _setinfo(space, w_info, "ftime()", 1e-3, False, True)
                 if return_ns:
                     return space.newint(
-                        widen(t.c_time) * 10**9 +
-                        widen(t.c_millitm) * 10**6)
+                        r_int64(t.c_time) * 10**9 +
+                        r_int64(t.c_millitm) * 10**6)
                 else:
                     return space.newfloat(
                         widen(t.c_time) +
@@ -327,7 +327,7 @@
                 _setinfo(space, w_info, "time()", 1.0, False, True)
             result = c_time(lltype.nullptr(rffi.TIME_TP.TO))
             if return_ns:
-                return space.newint(widen(result) * 10**9)
+                return space.newint(r_int64(result) * 10**9)
             else:
                 return space.newfloat(float(result))
     
@@ -821,7 +821,7 @@
         return widen(timespec.c_tv_sec) + widen(timespec.c_tv_nsec) * 1e-9
 
     def _timespec_to_nanoseconds(timespec):
-        return widen(timespec.c_tv_sec) * 10**9 + widen(timespec.c_tv_nsec)
+        return r_int64(timespec.c_tv_sec) * 10**9 + r_int64(timespec.c_tv_nsec)
     
     def _clock_gettime_impl(space, clk_id, return_ns):
         with lltype.scoped_alloc(TIMESPEC) as timespec:
@@ -861,7 +861,7 @@
             if ret != 0:
                 raise exception_from_saved_errno(space, space.w_OSError)
 
-    @unwrap_spec(clk_id='c_int', ns=int)
+    @unwrap_spec(clk_id='c_int', ns=r_int64)
     def clock_settime_ns(space, clk_id, ns):
         """clock_settime_ns(clk_id, time)
     
@@ -993,7 +993,7 @@
                 _setinfo(space, w_info, implementation, resolution, True, False)
 
             if return_ns:
-                return space.newint(widen(tick_count) * 10**6)
+                return space.newint(r_int64(tick_count) * 10**6)
             else:
                 return space.newfloat(tick_count * 1e-3)
 
@@ -1012,7 +1012,7 @@
             time = rffi.cast(lltype.Signed, c_mach_absolute_time())
             numer = rffi.getintfield(timebase_info, 'c_numer')
             denom = rffi.getintfield(timebase_info, 'c_denom')
-            nanosecs = time * numer / denom
+            nanosecs = r_int64(time) * numer / denom
             if w_info is not None:
                 res = (numer / denom) * 1e-9
                 _setinfo(space, w_info, "mach_absolute_time()", res, True, False)
@@ -1084,7 +1084,7 @@
                 _setinfo(space, w_info, "QueryPerformanceCounter()", resolution,
                          True, False)
             if return_ns:
-                return space.newint(widen(diff) * 10**9 // time_state.divisor)
+                return space.newint(r_int64(diff) * 10**9 // time_state.divisor)
             else:
                 return space.newfloat(float(diff) / float(time_state.divisor))
 
@@ -1139,7 +1139,7 @@
         if w_info is not None:
             _setinfo(space, w_info, "GetProcessTimes()", 1e-7, True, False)
         if return_ns:
-            return space.newint((widen(kernel_time2) + widen(user_time2)) * 10**2)
+            return space.newint((r_int64(kernel_time2) + r_int64(user_time2)) * 10**2)
         else:
             return space.newfloat((float(kernel_time2) + float(user_time2)) * 1e-7)
 else:
@@ -1198,20 +1198,20 @@
                                  1.0 / rposix.CLOCK_TICKS_PER_SECOND,
                                  True, False)
                     if return_ns:
-                        return space.newint(widen(cpu_time) * 10**9 // int(rposix.CLOCK_TICKS_PER_SECOND))
+                        return space.newint(r_int64(cpu_time) * 10**9 // int(rposix.CLOCK_TICKS_PER_SECOND))
                     else:
                         return space.newfloat(float(cpu_time) / rposix.CLOCK_TICKS_PER_SECOND)
         return _clock_impl(space, w_info, return_ns)
 
 def process_time(space, w_info=None):
     """process_time() -> float
-    
+
     Process time for profiling: sum of the kernel and user-space CPU time."""
     return _process_time_impl(space, w_info, False)
 
 def process_time_ns(space, w_info=None):
     """process_time() -> int
-    
+
     Process time for profiling as nanoseconds:
     sum of the kernel and user-space CPU time"""
     return _process_time_impl(space, w_info, True)
@@ -1232,7 +1232,7 @@
         _setinfo(space, w_info,
                  "clock()", 1.0 / CLOCKS_PER_SEC, True, False)
     if return_ns:
-        return space.newint(value * 10**9 // CLOCKS_PER_SEC)
+        return space.newint(r_int64(value) * 10**9 // CLOCKS_PER_SEC)
     else:
         return space.newfloat(float(value) / CLOCKS_PER_SEC)
 
diff --git a/rpython/rlib/rtime.py b/rpython/rlib/rtime.py
--- a/rpython/rlib/rtime.py
+++ b/rpython/rlib/rtime.py
@@ -9,7 +9,7 @@
 from rpython.rtyper.tool import rffi_platform
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib.objectmodel import register_replacement_for
-from rpython.rlib.rarithmetic import intmask, UINT_MAX, widen
+from rpython.rlib.rarithmetic import intmask, r_int64, UINT_MAX
 from rpython.rlib import rposix
 
 _WIN32 = sys.platform.startswith('win')
@@ -95,8 +95,8 @@
             float(rffi.getintfield(t, 'c_tv_usec')) * 0.000001)
 
 def decode_timeval_ns(t):
-    return (widen(rffi.getintfield(t, 'c_tv_sec')) * 10**9 +
-            widen(rffi.getintfield(t, 'c_tv_usec')) * 10**3)
+    return (r_int64(rffi.getintfield(t, 'c_tv_sec')) * 10**9 +
+            r_int64(rffi.getintfield(t, 'c_tv_usec')) * 10**3)
 
 
 def external(name, args, result, compilation_info=eci, **kwds):

From pypy.commits at gmail.com  Thu Nov  7 17:19:50 2019
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 07 Nov 2019 14:19:50 -0800 (PST)
Subject: [pypy-commit] pypy py3.6: Convert test_annotations.py into
 apptest_annotations.py
Message-ID: <5dc49886.1c69fb81.64c10.6b02@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6
Changeset: r97981:1c7a09311e66
Date: 2019-11-07 22:04 +0000
http://bitbucket.org/pypy/pypy/changeset/1c7a09311e66/

Log:	Convert test_annotations.py into apptest_annotations.py

diff --git a/pypy/interpreter/test/test_annotations.py b/pypy/interpreter/test/apptest_annotations.py
rename from pypy/interpreter/test/test_annotations.py
rename to pypy/interpreter/test/apptest_annotations.py
--- a/pypy/interpreter/test/test_annotations.py
+++ b/pypy/interpreter/test/apptest_annotations.py
@@ -1,168 +1,140 @@
-class AppTestAnnotations:
+def test_toplevel_annotation():
+    # exec because this needs to be in "top level" scope
+    exec("a: int; assert __annotations__['a'] == int")
 
-    def test_toplevel_annotation(self):
-        # exec because this needs to be in "top level" scope
-        # whereas the docstring-based tests are inside a function
-        # (or don't care)
-        exec("a: int; assert __annotations__['a'] == int")
+def test_toplevel_invalid():
+    exec('try: a: invalid\nexcept NameError: pass\n')
 
-    def test_toplevel_invalid(self):
-        exec('try: a: invalid\nexcept NameError: pass\n')
+def test_non_simple_annotation():
+    class C:
+        (a): int
+        assert "a" not in __annotations__
 
-    def test_non_simple_annotation(self):
-        '''
-        class C:
-            (a): int
-            assert "a" not in __annotations__
-        '''
+def test_simple_with_target():
+    class C:
+        a: int = 1
+        assert __annotations__["a"] == int
+        assert a == 1
 
-    def test_simple_with_target(self):
-        '''
-        class C:
-            a: int = 1
-            assert __annotations__["a"] == int
-            assert a == 1
-        '''
+def test_attribute_target():
+    class C:
+        a = 1
+        a.x: int
+        assert __annotations__ == {}
 
-    def test_attribute_target(self):
-        '''
-        class C:
-            a = 1
-            a.x: int
-            assert __annotations__ == {}
-        '''
+def test_subscript_target():
+    # ensure that these type annotations don't raise exceptions
+    # during compilation
+    class C:
+        a = 1
+        a[0]: int
+        a[1:2]: int
+        a[1:2:2]: int
+        a[1:2:2,...]: int
+        assert __annotations__ == {}
 
-    def test_subscript_target(self):
-        '''
-        # ensure that these type annotations don't raise exceptions
-        # during compilation
-        class C:
-            a = 1
-            a[0]: int
-            a[1:2]: int
-            a[1:2:2]: int
-            a[1:2:2,...]: int
-            assert __annotations__ == {}
-        '''
+def test_class_annotation():
+    class C:
+        a: int
+        b: str
+        assert "__annotations__" in locals()
+    assert C.__annotations__ == {"a": int, "b": str}
 
-    def test_class_annotation(self):
-        '''
-        class C:
+def test_unevaluated_name():
+    class C:
+        def __init__(self):
+            self.x: invalid_name = 1
+            assert self.x == 1
+    C()
+
+def test_nonexistent_target():
+    try:
+        # this is invalid because `y` is undefined
+        # it should raise a NameError
+        y[0]: invalid
+    except NameError:
+        ...
+
+def test_non_simple_func_annotation():
+    a = 5
+    def f():
+        (a): int
+        return a
+    assert f() == 5
+
+def test_repeated_setup():
+    # each exec will run another SETUP_ANNOTATIONS
+    # we want to confirm that this doesn't blow away
+    # the previous __annotations__
+    d = {}
+    exec('a: int', d)
+    exec('b: int', d)
+    exec('assert __annotations__ == {"a": int, "b": int}', d)
+
+def test_function_no___annotations__():
+    a: int
+    assert "__annotations__" not in locals()
+
+def test_unboundlocal():
+    # a simple variable annotation implies its target is a local
+    a: int
+    try:
+        print(a)
+    except UnboundLocalError:
+        return
+    assert False
+
+def test_ternary_expression_bug():
+    class C:
+        var: bool = True if False else False
+        assert var is False
+    assert C.__annotations__ == {"var": bool}
+
+def test_reassigned___annotations__():
+    class C:
+        __annotations__ = None
+        try:
             a: int
-            b: str
-            assert "__annotations__" in locals()
-        assert C.__annotations__ == {"a": int, "b": str}
-        '''
+            raise
+        except TypeError:
+            pass
+        except:
+            assert False
 
-    def test_unevaluated_name(self):
-        '''
-        class C:
-            def __init__(self):
-                self.x: invalid_name = 1
-                assert self.x == 1
-        C()
-        '''
+def test_locals_arent_dicts():
+    class O:
+        def __init__(self):
+            self.dct = {}
+        def __getitem__(self, name):
+            return self.dct[name]
+        def __setitem__(self, name, value):
+            self.dct[name] = value
+    # don't crash if locals aren't just a normal dict
+    exec("a: int; assert __annotations__['a'] == int", {}, O())
 
-    def test_nonexistent_target(self):
-        '''
-        try:
-            # this is invalid because `y` is undefined
-            # it should raise a NameError
-            y[0]: invalid
-        except NameError:
-            ...
-        '''
+def test_NameError_if_annotations_are_gone():
+    exec("""if 1:
+        raises(NameError, '''if 1:
+            class A:
+                del __annotations__
+                a: int
+        ''')
+    """)
 
-    def test_non_simple_func_annotation(self):
-        '''
-        a = 5
-        def f():
-            (a): int
-            return a
-        assert f() == 5
-        '''
-
-    def test_repeated_setup(self):
-        # each exec will run another SETUP_ANNOTATIONS
-        # we want to confirm that this doesn't blow away
-        # the previous __annotations__
-        d = {}
-        exec('a: int', d)
-        exec('b: int', d)
-        exec('assert __annotations__ == {"a": int, "b": int}', d)
-
-    def test_function_no___annotations__(self):
-        '''
-        a: int
-        assert "__annotations__" not in locals()
-        '''
-
-    def test_unboundlocal(self):
-        # a simple variable annotation implies its target is a local
-        '''
-        a: int
-        try:
-            print(a)
-        except UnboundLocalError:
-            return
-        assert False
-        '''
-
-    def test_ternary_expression_bug(self):
-        """
-        class C:
-            var: bool = True if False else False
-            assert var is False
-        assert C.__annotations__ == {"var": bool}
-        """
-
-    def test_reassigned___annotations__(self):
-        '''
-        class C:
-            __annotations__ = None
-            try:
-                a: int
-                raise
-            except TypeError:
-                pass
-            except:
-                assert False
-        '''
-
-    def test_locals_arent_dicts(self):
-        class O:
-            def __init__(self):
-                self.dct = {}
-            def __getitem__(self, name):
-                return self.dct[name]
-            def __setitem__(self, name, value):
-                self.dct[name] = value
-        # don't crash if locals aren't just a normal dict
-        exec("a: int; assert __annotations__['a'] == int", {}, O())
-
-    def test_NameError_if_annotations_are_gone(self):
-        exec("""if 1:
-            raises(NameError, '''if 1:
-                class A:
-                    del __annotations__
-                    a: int
-            ''')
-        """)
-
-    def test_lineno(self):
-        s = """
+def test_lineno():
+    s = """
 
 a: int
-        """
-        c = compile(s, "f", "exec")
-        assert c.co_firstlineno == 3
+    """
+    c = compile(s, "f", "exec")
+    assert c.co_firstlineno == 3
 
-    def test_scoping(self):
-        exec("""if 1:
-            def f(classvar):
-                class C:
-                    cls: classvar = 23
-                assert C.__annotations__ == {"cls": "abc"}
+def test_scoping():
+    exec("""if 1:
+        def f(classvar):
+            class C:
+                cls: classvar = 23
+            assert C.__annotations__ == {"cls": "abc"}
 
-            f("abc")
-        """)
+        f("abc")
+    """)

From pypy.commits at gmail.com  Thu Nov  7 17:19:53 2019
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 07 Nov 2019 14:19:53 -0800 (PST)
Subject: [pypy-commit] pypy py3.6: clean up some tests
Message-ID: <5dc49889.1c69fb81.eeca9.a488@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6
Changeset: r97982:537aeca90e39
Date: 2019-11-07 22:18 +0000
http://bitbucket.org/pypy/pypy/changeset/537aeca90e39/

Log:	clean up some tests

diff --git a/pypy/interpreter/test/apptest_annotations.py b/pypy/interpreter/test/apptest_annotations.py
--- a/pypy/interpreter/test/apptest_annotations.py
+++ b/pypy/interpreter/test/apptest_annotations.py
@@ -1,9 +1,17 @@
+import pytest
+
 def test_toplevel_annotation():
     # exec because this needs to be in "top level" scope
-    exec("a: int; assert __annotations__['a'] == int")
+    exec("""if True:
+        a: int
+        assert __annotations__['a'] == int
+    """)
 
 def test_toplevel_invalid():
-    exec('try: a: invalid\nexcept NameError: pass\n')
+    exec("""if True:
+        with pytest.raises(NameError):
+            a: invalid
+    """)
 
 def test_non_simple_annotation():
     class C:
@@ -30,7 +38,7 @@
         a[0]: int
         a[1:2]: int
         a[1:2:2]: int
-        a[1:2:2,...]: int
+        a[1:2:2, ...]: int
         assert __annotations__ == {}
 
 def test_class_annotation():
@@ -78,11 +86,8 @@
 def test_unboundlocal():
     # a simple variable annotation implies its target is a local
     a: int
-    try:
+    with pytest.raises(UnboundLocalError):
         print(a)
-    except UnboundLocalError:
-        return
-    assert False
 
 def test_ternary_expression_bug():
     class C:
@@ -93,33 +98,28 @@
 def test_reassigned___annotations__():
     class C:
         __annotations__ = None
-        try:
+        with pytest.raises(TypeError):
             a: int
-            raise
-        except TypeError:
-            pass
-        except:
-            assert False
 
 def test_locals_arent_dicts():
     class O:
         def __init__(self):
             self.dct = {}
+
         def __getitem__(self, name):
             return self.dct[name]
+
         def __setitem__(self, name, value):
             self.dct[name] = value
+
     # don't crash if locals aren't just a normal dict
     exec("a: int; assert __annotations__['a'] == int", {}, O())
 
 def test_NameError_if_annotations_are_gone():
-    exec("""if 1:
-        raises(NameError, '''if 1:
-            class A:
-                del __annotations__
-                a: int
-        ''')
-    """)
+    with pytest.raises(NameError):
+        class A:
+            del __annotations__
+            a: int
 
 def test_lineno():
     s = """
@@ -130,11 +130,9 @@
     assert c.co_firstlineno == 3
 
 def test_scoping():
-    exec("""if 1:
-        def f(classvar):
-            class C:
-                cls: classvar = 23
-            assert C.__annotations__ == {"cls": "abc"}
+    def f(classvar):
+        class C:
+            cls: classvar = 23
+        assert C.__annotations__ == {"cls": "abc"}
 
-        f("abc")
-    """)
+    f("abc")

From pypy.commits at gmail.com  Thu Nov  7 18:56:14 2019
From: pypy.commits at gmail.com (wlav)
Date: Thu, 07 Nov 2019 15:56:14 -0800 (PST)
Subject: [pypy-commit] pypy cppyy-dev: use cling-config to get the cpp flags
Message-ID: <5dc4af1e.1c69fb81.33a4f.978b@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-dev
Changeset: r97984:8d5616349f10
Date: 2019-11-06 19:06 -0800
http://bitbucket.org/pypy/pypy/changeset/8d5616349f10/

Log:	use cling-config to get the cpp flags

diff --git a/pypy/module/_cppyy/test/Makefile b/pypy/module/_cppyy/test/Makefile
--- a/pypy/module/_cppyy/test/Makefile
+++ b/pypy/module/_cppyy/test/Makefile
@@ -15,7 +15,7 @@
 
 HASGENREFLEX:=$(shell command -v genreflex 2> /dev/null)
 
-cppflags=-std=c++14 -O3 -fPIC -rdynamic
+cppflags=$(shell cling-config --cppflags) -O3 -fPIC -rdynamic
 ifdef HASGENREFLEX
   genreflex_flags:=$(shell genreflex --cppflags)
   cppflags+=$(genreflex_flags)

From pypy.commits at gmail.com  Thu Nov  7 18:56:16 2019
From: pypy.commits at gmail.com (wlav)
Date: Thu, 07 Nov 2019 15:56:16 -0800 (PST)
Subject: [pypy-commit] pypy cppyy-dev: add test_regression file with
 regression tests
Message-ID: <5dc4af20.1c69fb81.57f22.ac62@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-dev
Changeset: r97985:7c7fd9713e04
Date: 2019-11-06 19:06 -0800
http://bitbucket.org/pypy/pypy/changeset/7c7fd9713e04/

Log:	add test_regression file with regression tests

diff --git a/pypy/module/_cppyy/test/test_regression.py b/pypy/module/_cppyy/test/test_regression.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_cppyy/test/test_regression.py
@@ -0,0 +1,50 @@
+import py, os, sys
+from .support import setup_make
+
+from pypy.module._cppyy import interp_cppyy, executor
+
+
+class AppTestREGRESSION:
+    spaceconfig = dict(usemodules=['_cppyy', '_rawffi', 'itertools'])
+
+    def setup_class(cls):
+        cls.w_example01 = cls.space.appexec([], """():
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()""")
+
+    def test01_dir(self):
+        """These dir() methods used to crash."""
+
+        import _cppyy as cppyy
+
+        cppyy.gbl.gInterpreter.Declare("namespace cppyy_regression_test { void iii() {}; }")
+
+        assert not 'iii' in cppyy.gbl.cppyy_regression_test.__dict__
+        assert not '__abstractmethods__' in dir(cppyy.gbl.cppyy_regression_test)
+        assert '__class__' in dir(cppyy.gbl.cppyy_regression_test)
+        assert 'iii' in dir(cppyy.gbl.cppyy_regression_test)
+
+        assert not 'iii' in cppyy.gbl.cppyy_regression_test.__dict__
+        assert cppyy.gbl.cppyy_regression_test.iii
+        assert 'iii' in cppyy.gbl.cppyy_regression_test.__dict__
+
+    def test02_default_template_arguments(self):
+        """Calling a templated method on a templated class with all defaults used to crash."""
+
+        import _cppyy as cppyy
+
+        cppyy.gbl.gInterpreter.Declare("""
+        template<typename T>
+        class AllDefault {
+        public:
+           AllDefault(int val) : m_t(val) {}
+           template<int aap=1, int noot=2>
+              int do_stuff() { return m_t+aap+noot; }
+
+        public:
+           T m_t;
+        };""")
+
+        a = cppyy.gbl.AllDefault[int](24)
+        a.m_t = 21;
+        assert a.do_stuff() == 24

From pypy.commits at gmail.com  Thu Nov  7 18:56:17 2019
From: pypy.commits at gmail.com (wlav)
Date: Thu, 07 Nov 2019 15:56:17 -0800 (PST)
Subject: [pypy-commit] pypy cppyy-dev: fix backend name selection
Message-ID: <5dc4af21.1c69fb81.72367.fee2@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-dev
Changeset: r97986:971d12f69dba
Date: 2019-11-06 19:41 -0800
http://bitbucket.org/pypy/pypy/changeset/971d12f69dba/

Log:	fix backend name selection

diff --git a/pypy/module/_cppyy/capi/loadable_capi.py b/pypy/module/_cppyy/capi/loadable_capi.py
--- a/pypy/module/_cppyy/capi/loadable_capi.py
+++ b/pypy/module/_cppyy/capi/loadable_capi.py
@@ -1,4 +1,4 @@
-import os
+import os, sys
 
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib.rarithmetic import intmask
@@ -20,8 +20,10 @@
 from pypy.module._cppyy.capi.capi_types import C_SCOPE, C_TYPE, C_OBJECT,\
    C_METHOD, C_INDEX, C_INDEX_ARRAY, WLAVC_INDEX, C_FUNC_PTR
 
-
-backend_library = 'libcppyy_backend.so'
+backend_ext = '.so'
+if 'win32' in sys.platform:
+    backend_ext = '.dll'
+backend_library = 'libcppyy_backend'
 
 # this is not technically correct, but will do for now
 std_string_name = 'std::basic_string<char>'
@@ -311,7 +313,11 @@
             state.backend = W_Library(space, space.newtext(libname), dldflags)
         else:
             # try usual lookups
-            state.backend = W_Library(space, space.newtext(backend_library), dldflags)
+            try:
+                state.backend = W_Library(space, space.newtext(backend_library+backend_ext), dldflags)
+            except Exception:
+                # TODO: where to find the value '.pypy-41'? Note that this only matters for testing.
+                state.backend = W_Library(space, space.newtext(backend_library+'.pypy-41'+backend_ext), dldflags)
 
         if state.backend:
             # fix constants

From pypy.commits at gmail.com  Thu Nov  7 18:56:19 2019
From: pypy.commits at gmail.com (wlav)
Date: Thu, 07 Nov 2019 15:56:19 -0800 (PST)
Subject: [pypy-commit] pypy cppyy-dev: bring capi to 1.10.6
Message-ID: <5dc4af23.1c69fb81.ac2ee.6643@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-dev
Changeset: r97987:dbc11284f2b1
Date: 2019-11-06 22:41 -0800
http://bitbucket.org/pypy/pypy/changeset/dbc11284f2b1/

Log:	bring capi to 1.10.6

diff --git a/pypy/module/_cppyy/capi/loadable_capi.py b/pypy/module/_cppyy/capi/loadable_capi.py
--- a/pypy/module/_cppyy/capi/loadable_capi.py
+++ b/pypy/module/_cppyy/capi/loadable_capi.py
@@ -43,6 +43,8 @@
     def __init__(self, val):
         _Arg.__init__(self, 'h', h = val)
 
+_ArgU = _ArgH      # simple re-use for indices (size_t)
+
 class _ArgL(_Arg):
     _immutable_ = True
     def __init__(self, val):
@@ -144,13 +146,17 @@
 
         # TODO: the following need to match up with the globally defined C_XYZ low-level
         # types (see capi/__init__.py), but by using strings here, that isn't guaranteed
-        c_opaque_ptr = state.c_ulong    # not ptrdiff_t (which is signed)
+        c_opaque_ptr = state.c_uintptr_t    # not size_t (which is signed)
+        c_size_t     = state.c_size_t
+        c_ptrdiff_t  = state.c_ptrdiff_t
+        c_intptr_t   = state.c_intptr_t
+        c_uintptr_t  = state.c_uintptr_t
  
         c_scope       = c_opaque_ptr
         c_type        = c_scope
-        c_object      = c_opaque_ptr    # not voidp (to stick with one handle type)
-        c_method      = c_opaque_ptr
-        c_index       = state.c_long
+        c_object      = c_opaque_ptr        # not voidp (to stick with one handle type)
+        c_method      = c_uintptr_t         # not intptr_t (which is signed)
+        c_index       = c_size_t
         c_index_array = state.c_voidp
 
         c_void    = state.c_void
@@ -168,10 +174,10 @@
         c_ccharp = state.c_ccharp
         c_voidp  = state.c_voidp
 
-        c_size_t = nt.new_primitive_type(space, 'size_t')
-        c_ptrdiff_t = nt.new_primitive_type(space, 'ptrdiff_t')
+        self.capi_call_ifaces = {
+            # direct interpreter access
+            'compile'                  : ([c_ccharp],                 c_int),
 
-        self.capi_call_ifaces = {
             # name to opaque C++ scope representation
             'resolve_name'             : ([c_ccharp],                 c_ccharp),
             'resolve_enum'             : ([c_ccharp],                 c_ccharp),
@@ -223,12 +229,17 @@
 
             'get_all_cpp_names'        : ([c_scope, c_voidp],         c_voidp), # const char**
 
+            # namespace reflection information
+            'get_using_namespaces'     : ([c_scope],                  c_index),
+
             # type/class reflection information
             'final_name'               : ([c_type],                   c_ccharp),
             'scoped_final_name'        : ([c_type],                   c_ccharp),
+            'has_virtual_destructor'   : ([c_type],                   c_int),
             'has_complex_hierarchy'    : ([c_type],                   c_int),
             'num_bases'                : ([c_type],                   c_int),
             'base_name'                : ([c_type, c_int],            c_ccharp),
+            'is_smartptr'              : ([c_type],                   c_int),
             'is_subtype'               : ([c_type, c_type],           c_int),
             'smartptr_info'            : ([c_ccharp, c_voidp, c_voidp],         c_int),
             'add_smartptr_type'        : ([c_ccharp],                 c_void),
@@ -247,9 +258,11 @@
             'method_result_type'       : ([c_method],                 c_ccharp),
             'method_num_args'          : ([c_method],                 c_int),
             'method_req_args'          : ([c_method],                 c_int),
+            'method_arg_name'          : ([c_method, c_int],          c_ccharp),
             'method_arg_type'          : ([c_method, c_int],          c_ccharp),
             'method_arg_default'       : ([c_method, c_int],          c_ccharp),
             'method_signature'         : ([c_method, c_int],          c_ccharp),
+            'method_signature_max'     : ([c_method, c_int, c_int],   c_ccharp),
             'method_prototype'         : ([c_scope, c_method, c_int], c_ccharp),
             'is_const_method'          : ([c_method],                 c_int),
 
@@ -271,7 +284,7 @@
             'num_datamembers'          : ([c_scope],                  c_int),
             'datamember_name'          : ([c_scope, c_int],           c_ccharp),
             'datamember_type'          : ([c_scope, c_int],           c_ccharp),
-            'datamember_offset'        : ([c_scope, c_int],           c_ptrdiff_t),
+            'datamember_offset'        : ([c_scope, c_int],           c_intptr_t),
             'datamember_index'         : ([c_scope, c_ccharp],        c_int),
 
             # data member properties
@@ -360,7 +373,10 @@
     return rffi.cast(rffi.SIZE_T, space.uint_w(w_cdata))
 
 def _cdata_to_ptrdiff_t(space, w_cdata):
-    return rffi.cast(rffi.LONG, space.int_w(w_cdata))
+    return rffi.cast(rffi.PTRDIFF_T, space.int_w(w_cdata))
+
+def _cdata_to_intptr_t(space, w_cdata):
+    return rffi.cast(rffi.INTPTR_T, space.int_w(w_cdata))
 
 def _cdata_to_ptr(space, w_cdata): # TODO: this is both a hack and dreadfully slow
     w_cdata = space.interp_w(cdataobj.W_CData, w_cdata, can_be_None=False)
@@ -371,6 +387,10 @@
     ptr = _cdata_to_ptr(space, w_cdata)      # see above ... something better?
     return rffi.cast(rffi.CCHARP, ptr)
 
+# direct interpreter access
+def c_compile(space, code):
+    return space.int_w(call_capi(space, 'compile', [_ArgS(code)]))
+
 # name to opaque C++ scope representation ------------------------------------
 def c_resolve_name(space, name):
     return charp2str_free(space, call_capi(space, 'resolve_name', [_ArgS(name)]))
@@ -494,11 +514,17 @@
     c_free(space, rffi.cast(rffi.VOIDP, rawnames))        # id.
     return allnames
 
+# namespace reflection information
+def c_get_using_namespaces(space, cppscope):
+    return space.uint_w(call_capi(space, 'get_using_namespaces', [_ArgH(cppscope)]))
+
 # type/class reflection information ------------------------------------------
 def c_final_name(space, cpptype):
     return charp2str_free(space, call_capi(space, 'final_name', [_ArgH(cpptype)]))
 def c_scoped_final_name(space, cpptype):
     return charp2str_free(space, call_capi(space, 'scoped_final_name', [_ArgH(cpptype)]))
+def c_has_virtual_destructor(space, handle):
+    return space.bool_w(call_capi(space, 'has_virtual_destructor', [_ArgH(handle)]))
 def c_has_complex_hierarchy(space, handle):
     return space.bool_w(call_capi(space, 'has_complex_hierarchy', [_ArgH(handle)]))
 def c_num_bases(space, cppclass):
@@ -506,6 +532,8 @@
 def c_base_name(space, cppclass, base_index):
     args = [_ArgH(cppclass.handle), _ArgL(base_index)]
     return charp2str_free(space, call_capi(space, 'base_name', args))
+def c_is_smartptr(space, handle):
+    return space.bool_w(call_capi(space, 'is_smartptr', [_ArgH(handle)]))
 def c_is_subtype(space, derived, base):
     jit.promote(base)
     if derived == base:
@@ -558,7 +586,7 @@
     return py_indices
 
 def c_get_method(space, cppscope, index):
-    args = [_ArgH(cppscope.handle), _ArgL(index)]
+    args = [_ArgH(cppscope.handle), _ArgU(index)]
     return rffi.cast(C_METHOD, space.uint_w(call_capi(space, 'get_method', args)))
 
 def c_method_name(space, cppmeth):
@@ -573,6 +601,9 @@
     return space.int_w(call_capi(space, 'method_num_args', [_ArgH(cppmeth)]))
 def c_method_req_args(space, cppmeth):
     return space.int_w(call_capi(space, 'method_req_args', [_ArgH(cppmeth)]))
+def c_method_arg_name(space, cppmeth, arg_index):
+    args = [_ArgH(cppmeth), _ArgL(arg_index)]
+    return charp2str_free(space, call_capi(space, 'method_arg_name', args))
 def c_method_arg_type(space, cppmeth, arg_index):
     args = [_ArgH(cppmeth), _ArgL(arg_index)]
     return charp2str_free(space, call_capi(space, 'method_arg_type', args))
@@ -582,6 +613,9 @@
 def c_method_signature(space, cppmeth, show_formalargs=True):
     args = [_ArgH(cppmeth), _ArgL(show_formalargs)]
     return charp2str_free(space, call_capi(space, 'method_signature', args))
+def c_method_signature_max(space, cppmeth, show_formalargs, maxargs):
+    args = [_ArgH(cppmeth), _ArgL(show_formalargs), _ArgL(maxargs)]
+    return charp2str_free(space, call_capi(space, 'method_signature_max', args))
 def c_method_prototype(space, cppscope, cppmeth, show_formalargs=True):
     args = [_ArgH(cppscope.handle), _ArgH(cppmeth), _ArgL(show_formalargs)]
     return charp2str_free(space, call_capi(space, 'method_prototype', args))
@@ -589,24 +623,24 @@
     return space.bool_w(call_capi(space, 'is_const_method', [_ArgH(cppmeth)]))
 
 def c_get_num_templated_methods(space, cppscope):
-    return space.int_w(call_capi(space, 'method_is_template', [_ArgH(cppscope.handle)]))
+    return space.int_w(call_capi(space, 'get_num_templated_methods', [_ArgH(cppscope.handle)]))
 def c_get_templated_method_name(space, cppscope, index):
-    args = [_ArgH(cppscope.handle), _ArgL(index)]
-    return charp2str_free(space, call_capi(space, 'method_is_template', args))
+    args = [_ArgH(cppscope.handle), _ArgU(index)]
+    return charp2str_free(space, call_capi(space, 'get_templated_method_name', args))
 def c_exists_method_template(space, cppscope, name):
     args = [_ArgH(cppscope.handle), _ArgS(name)]
     return space.bool_w(call_capi(space, 'exists_method_template', args))
 def c_method_is_template(space, cppscope, index):
-    args = [_ArgH(cppscope.handle), _ArgL(index)]
+    args = [_ArgH(cppscope.handle), _ArgU(index)]
     return space.bool_w(call_capi(space, 'method_is_template', args))
 def c_get_method_template(space, cppscope, name, proto):
     args = [_ArgH(cppscope.handle), _ArgS(name), _ArgS(proto)]
-    return rffi.cast(C_METHOD, space.uint_w(call_capi(space, 'get_method_template', args)))
+    return rffi.cast(C_METHOD, space.int_w(call_capi(space, 'get_method_template', args)))
 
 def c_get_global_operator(space, nss, lc, rc, op):
     if nss is not None:
         args = [_ArgH(nss.handle), _ArgH(lc.handle), _ArgH(rc.handle), _ArgS(op)]
-        return rffi.cast(WLAVC_INDEX, space.int_w(call_capi(space, 'get_global_operator', args)))
+        return rffi.cast(WLAVC_INDEX, space.uint_w(call_capi(space, 'get_global_operator', args)))
     return rffi.cast(WLAVC_INDEX, -1)
 
 # method properties ----------------------------------------------------------
@@ -630,7 +664,7 @@
     return  charp2str_free(space, call_capi(space, 'datamember_type', args))
 def c_datamember_offset(space, cppscope, datamember_index):
     args = [_ArgH(cppscope.handle), _ArgL(datamember_index)]
-    return _cdata_to_ptrdiff_t(space, call_capi(space, 'datamember_offset', args))
+    return _cdata_to_intptr_t(space, call_capi(space, 'datamember_offset', args))
 
 def c_datamember_index(space, cppscope, name):
     args = [_ArgH(cppscope.handle), _ArgS(name)]
diff --git a/pypy/module/_cppyy/include/capi.h b/pypy/module/_cppyy/include/capi.h
--- a/pypy/module/_cppyy/include/capi.h
+++ b/pypy/module/_cppyy/include/capi.h
@@ -8,16 +8,19 @@
 extern "C" {
 #endif // ifdef __cplusplus
 
-    typedef ptrdiff_t     cppyy_scope_t;
+    typedef size_t        cppyy_scope_t;
     typedef cppyy_scope_t cppyy_type_t;
     typedef void*         cppyy_object_t;
-    typedef ptrdiff_t     cppyy_method_t;
+    typedef intptr_t      cppyy_method_t;
 
-    typedef long          cppyy_index_t;
+    typedef size_t        cppyy_index_t;
     typedef void*         cppyy_funcaddr_t;
 
     typedef unsigned long cppyy_exctype_t;
 
+    /* direct interpreter access ---------------------------------------------- */
+    int cppyy_compile(const char* code);
+
     /* name to opaque C++ scope representation -------------------------------- */
     RPY_EXTERN
     char* cppyy_resolve_name(const char* cppitem_name);
@@ -103,12 +106,17 @@
     RPY_EXTERN
     const char** cppyy_get_all_cpp_names(cppyy_scope_t scope, size_t* count);
 
+    /* namespace reflection information --------------------------------------- */
+    cppyy_index_t* cppyy_get_using_namespaces(cppyy_scope_t scope);
+
     /* class reflection information ------------------------------------------- */
     RPY_EXTERN
     char* cppyy_final_name(cppyy_type_t type);
     RPY_EXTERN
     char* cppyy_scoped_final_name(cppyy_type_t type);
     RPY_EXTERN
+    int cppyy_has_virtual_destructor(cppyy_type_t type);
+    RPY_EXTERN
     int cppyy_has_complex_hierarchy(cppyy_type_t type);
     RPY_EXTERN
     int cppyy_num_bases(cppyy_type_t type);
@@ -117,6 +125,8 @@
     RPY_EXTERN
     int cppyy_is_subtype(cppyy_type_t derived, cppyy_type_t base);
     RPY_EXTERN
+    int cppyy_is_smartptr(cppyy_type_t type);
+    RPY_EXTERN
     int cppyy_smartptr_info(const char* name, cppyy_type_t* raw, cppyy_method_t* deref);
     RPY_EXTERN
     void cppyy_add_smartptr_type(const char* type_name);
@@ -147,12 +157,16 @@
     RPY_EXTERN
     int cppyy_method_req_args(cppyy_method_t);
     RPY_EXTERN
+    char* cppyy_method_arg_name(cppyy_method_t,int arg_index);
+    RPY_EXTERN
     char* cppyy_method_arg_type(cppyy_method_t, int arg_index);
     RPY_EXTERN
     char* cppyy_method_arg_default(cppyy_method_t, int arg_index);
     RPY_EXTERN
     char* cppyy_method_signature(cppyy_method_t, int show_formalargs);
     RPY_EXTERN
+    char* cppyy_method_signature_max(cppyy_method_t, int show_formalargs, int maxargs);
+    RPY_EXTERN
     char* cppyy_method_prototype(cppyy_scope_t scope, cppyy_method_t, int show_formalargs);
     RPY_EXTERN
     int cppyy_is_const_method(cppyy_method_t);
@@ -190,7 +204,7 @@
     RPY_EXTERN
     char* cppyy_datamember_type(cppyy_scope_t scope, int datamember_index);
     RPY_EXTERN
-    ptrdiff_t cppyy_datamember_offset(cppyy_scope_t scope, int datamember_index);
+    intptr_t cppyy_datamember_offset(cppyy_scope_t scope, int datamember_index);
     RPY_EXTERN
     int cppyy_datamember_index(cppyy_scope_t scope, const char* name);
 

From pypy.commits at gmail.com  Thu Nov  7 18:56:21 2019
From: pypy.commits at gmail.com (wlav)
Date: Thu, 07 Nov 2019 15:56:21 -0800 (PST)
Subject: [pypy-commit] pypy cppyy-dev: add new builtin types needed for
 1.10.6
Message-ID: <5dc4af25.1c69fb81.29693.7182@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-dev
Changeset: r97988:ef9d9c6bea09
Date: 2019-11-06 22:43 -0800
http://bitbucket.org/pypy/pypy/changeset/ef9d9c6bea09/

Log:	add new builtin types needed for 1.10.6

diff --git a/pypy/module/_cppyy/ffitypes.py b/pypy/module/_cppyy/ffitypes.py
--- a/pypy/module/_cppyy/ffitypes.py
+++ b/pypy/module/_cppyy/ffitypes.py
@@ -38,6 +38,8 @@
         # special types
         self.c_size_t    = nt.new_primitive_type(space, 'size_t')
         self.c_ptrdiff_t = nt.new_primitive_type(space, 'ptrdiff_t')
+        self.c_intptr_t  = nt.new_primitive_type(space, 'intptr_t')
+        self.c_uintptr_t = nt.new_primitive_type(space, 'uintptr_t')
 
 class BoolTypeMixin(object):
     _mixin_     = True

From pypy.commits at gmail.com  Thu Nov  7 18:56:22 2019
From: pypy.commits at gmail.com (wlav)
Date: Thu, 07 Nov 2019 15:56:22 -0800 (PST)
Subject: [pypy-commit] pypy cppyy-dev: simplify iteration over std::list/map
Message-ID: <5dc4af26.1c69fb81.ec714.6e67@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-dev
Changeset: r97989:466fe5506d3d
Date: 2019-11-07 11:37 -0800
http://bitbucket.org/pypy/pypy/changeset/466fe5506d3d/

Log:	simplify iteration over std::list/map

diff --git a/pypy/module/_cppyy/pythonify.py b/pypy/module/_cppyy/pythonify.py
--- a/pypy/module/_cppyy/pythonify.py
+++ b/pypy/module/_cppyy/pythonify.py
@@ -439,14 +439,17 @@
     # also the fallback on the indexed __getitem__, but that is slower)
     add_checked_item = False
     if name.find('std::vector', 0, 11) != 0:
-        if ('begin' in pyclass.__dict__ and 'end' in pyclass.__dict__):
+        if 'begin' in pyclass.__dict__ and 'end' in pyclass.__dict__:
             if _cppyy._scope_byname(name+'::iterator') or \
                     _cppyy._scope_byname(name+'::const_iterator'):
                 def __iter__(self):
                     i = self.begin()
-                    while i != self.end():
+                    end = self.size()
+                    count = 0
+                    while count != end:
                         yield i.__deref__()
                         i.__preinc__()
+                        count += 1
                     i.__destruct__()
                     raise StopIteration
                 pyclass.__iter__ = __iter__

From pypy.commits at gmail.com  Thu Nov  7 18:56:24 2019
From: pypy.commits at gmail.com (wlav)
Date: Thu, 07 Nov 2019 15:56:24 -0800 (PST)
Subject: [pypy-commit] pypy cppyy-dev: fix string naming (basic_string<char>
 -> string)
Message-ID: <5dc4af28.1c69fb81.8bc5c.ebff@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-dev
Changeset: r97990:78794b47737d
Date: 2019-11-07 15:20 -0800
http://bitbucket.org/pypy/pypy/changeset/78794b47737d/

Log:	fix string naming (basic_string<char> -> string)

diff --git a/pypy/module/_cppyy/capi/loadable_capi.py b/pypy/module/_cppyy/capi/loadable_capi.py
--- a/pypy/module/_cppyy/capi/loadable_capi.py
+++ b/pypy/module/_cppyy/capi/loadable_capi.py
@@ -26,7 +26,7 @@
 backend_library = 'libcppyy_backend'
 
 # this is not technically correct, but will do for now
-std_string_name = 'std::basic_string<char>'
+std_string_name = 'std::string'
 
 class _Arg:         # poor man's union
     _immutable_ = True
diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py
--- a/pypy/module/_cppyy/converter.py
+++ b/pypy/module/_cppyy/converter.py
@@ -993,7 +993,7 @@
 _converters["void*&"]                   = VoidPtrRefConverter
 
 # special cases (note: 'string' aliases added below)
-_converters["std::basic_string<char>"]           = StdStringConverter
+_converters["std::string"]                       = StdStringConverter
 _converters["const std::basic_string<char>&"]    = StdStringConverter     # TODO: shouldn't copy
 _converters["std::basic_string<char>&"]          = StdStringRefConverter
 _converters["std::basic_string<char>&&"]         = StdStringMoveConverter
@@ -1126,7 +1126,8 @@
         ("char",                            "signed char"),   # TODO: check
         ("const char*",                     "char*"),
 
-        ("std::basic_string<char>",         "string"),
+        ("std::string",                     "string"),
+        ("std::string",                     "std::basic_string<char>"),
         ("const std::basic_string<char>&",  "const string&"),
         ("std::basic_string<char>&",        "string&"),
         ("std::basic_string<char>&&",       "string&&"),
diff --git a/pypy/module/_cppyy/executor.py b/pypy/module/_cppyy/executor.py
--- a/pypy/module/_cppyy/executor.py
+++ b/pypy/module/_cppyy/executor.py
@@ -384,7 +384,7 @@
 # special cases (note: 'string' aliases added below)
 _executors["constructor"]         = ConstructorExecutor
 
-_executors["std::basic_string<char>"]         = StdStringExecutor
+_executors["std::string"]         = StdStringExecutor
 _executors["const std::basic_string<char>&"]  = StdStringRefExecutor
 _executors["std::basic_string<char>&"]        = StdStringRefExecutor
 
@@ -453,7 +453,8 @@
     aliases = (
         ("const char*",                     "char*"),
 
-        ("std::basic_string<char>",         "string"),
+        ("std::string",                     "string"),
+        ("std::string",                     "std::basic_string<char>"),
         ("const std::basic_string<char>&",  "const string&"),
         ("std::basic_string<char>&",        "string&"),
 

From pypy.commits at gmail.com  Thu Nov  7 18:56:25 2019
From: pypy.commits at gmail.com (wlav)
Date: Thu, 07 Nov 2019 15:56:25 -0800 (PST)
Subject: [pypy-commit] pypy cppyy-dev: rtype fixers to make test_zjit run
Message-ID: <5dc4af29.1c69fb81.33c90.0dd4@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-dev
Changeset: r97991:8f3cf9c3225b
Date: 2019-11-07 15:54 -0800
http://bitbucket.org/pypy/pypy/changeset/8f3cf9c3225b/

Log:	rtype fixers to make test_zjit run

diff --git a/pypy/module/_cppyy/capi/loadable_capi.py b/pypy/module/_cppyy/capi/loadable_capi.py
--- a/pypy/module/_cppyy/capi/loadable_capi.py
+++ b/pypy/module/_cppyy/capi/loadable_capi.py
@@ -30,9 +30,10 @@
 
 class _Arg:         # poor man's union
     _immutable_ = True
-    def __init__(self, tc, h = 0, l = -1, d = -1., s = '', p = rffi.cast(rffi.VOIDP, 0)):
+    def __init__(self, tc, h = 0, u = 0, l = -1, d = -1., s = '', p = rffi.cast(rffi.VOIDP, 0)):
         self.tc      = tc
         self._handle = h
+        self._index  = u
         self._long   = l
         self._double = d
         self._string = s
@@ -43,7 +44,10 @@
     def __init__(self, val):
         _Arg.__init__(self, 'h', h = val)
 
-_ArgU = _ArgH      # simple re-use for indices (size_t)
+class _ArgU(_Arg):      # separate class for rtyper
+    _immutable_ = True
+    def __init__(self, val):
+        _Arg.__init__(self, 'u', u = val)
 
 class _ArgL(_Arg):
     _immutable_ = True
@@ -99,7 +103,10 @@
                     misc.write_raw_signed_data(data, rffi.cast(rffi.LONG, obj._long), argtype.size)
                 elif obj.tc == 'h':
                     assert isinstance(argtype, ctypeprim.W_CTypePrimitiveUnsigned)
-                    misc.write_raw_unsigned_data(data, rffi.cast(rffi.ULONG, obj._handle), argtype.size)
+                    misc.write_raw_unsigned_data(data, rffi.cast(rffi.UINTPTR_T, obj._handle), argtype.size)
+                elif obj.tc == 'u':
+                    assert isinstance(argtype, ctypeprim.W_CTypePrimitiveUnsigned)
+                    misc.write_raw_unsigned_data(data, rffi.cast(rffi.SIZE_T, obj._index), argtype.size)
                 elif obj.tc == 'p':
                     assert obj._voidp != rffi.cast(rffi.VOIDP, 0)
                     data = rffi.cast(rffi.VOIDPP, data)
diff --git a/pypy/module/_cppyy/test/test_zjit.py b/pypy/module/_cppyy/test/test_zjit.py
--- a/pypy/module/_cppyy/test/test_zjit.py
+++ b/pypy/module/_cppyy/test/test_zjit.py
@@ -230,6 +230,9 @@
         assert isinstance(w_obj, FakeString)
         return w_obj.val
 
+    def fsencode_w(self, w_obj):
+        return self.bytes_w(w_obj)
+
     def text_w(self, w_obj):
         assert isinstance(w_obj, FakeString)
         return w_obj.val

From pypy.commits at gmail.com  Fri Nov  8 02:18:15 2019
From: pypy.commits at gmail.com (arigo)
Date: Thu, 07 Nov 2019 23:18:15 -0800 (PST)
Subject: [pypy-commit] pypy py3.6: oops sorry.  Test and fix
Message-ID: <5dc516b7.1c69fb81.51f9f.071e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.6
Changeset: r97992:a0ad758e734a
Date: 2019-11-08 08:17 +0100
http://bitbucket.org/pypy/pypy/changeset/a0ad758e734a/

Log:	oops sorry. Test and fix

diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -262,6 +262,8 @@
         self.sys_exc_operror = operror
 
     def set_sys_exc_info3(self, w_type, w_value, w_traceback):
+        from pypy.interpreter import pytraceback
+
         space = self.space
         if space.is_none(w_value):
             operror = None
diff --git a/pypy/module/__pypy__/test/test_magic.py b/pypy/module/__pypy__/test/test_magic.py
--- a/pypy/module/__pypy__/test/test_magic.py
+++ b/pypy/module/__pypy__/test/test_magic.py
@@ -77,3 +77,18 @@
         except RuntimeError as e:
             assert e.__cause__ is None
             assert e.__context__ is None
+
+    def test_set_exc_info_traceback(self):
+        import sys
+        from __pypy__ import set_exc_info
+        def f():
+            1 // 0
+        def g():
+            try:
+                f()
+            except ZeroDivisionError:
+                return sys.exc_info()[2]
+        tb = g()
+        terr = TypeError("hello world")
+        set_exc_info(TypeError, terr, tb)
+        assert sys.exc_info()[2] is tb
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -2,7 +2,6 @@
 
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.interpreter.error import OperationError, oefmt, strerror as _strerror
-from pypy.interpreter import pytraceback
 from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, CONST_STRING
 from pypy.module.cpyext.api import PyObjectFields, cpython_struct
 from pypy.module.cpyext.api import bootstrap_function, slot_function

From pypy.commits at gmail.com  Fri Nov  8 02:26:24 2019
From: pypy.commits at gmail.com (arigo)
Date: Thu, 07 Nov 2019 23:26:24 -0800 (PST)
Subject: [pypy-commit] pypy py3.7-pep564: - fix obscure annotation issue by
 using r_int64(intmask(t.c_millitm)) instead
Message-ID: <5dc518a0.1c69fb81.9c9cf.493e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.7-pep564
Changeset: r97993:3f033e498621
Date: 2019-11-08 08:25 +0100
http://bitbucket.org/pypy/pypy/changeset/3f033e498621/

Log:	- fix obscure annotation issue by using
	r_int64(intmask(t.c_millitm)) instead of r_int64(t.c_millitm). On
	64-bit machines the latter is equivalent to int(t.c_millitm), and
	c_millitm is unsigned, which makes the annotator think we should use
	intmask() instead of int().

	- use space.newint() instead of space.newlong_from_rarith_int()
	again. Sorry! I realized that space.newint() would accept any
	integer type, and fall back automatically to from_rarith_int()
	depending on the type. This is better because on 64-bit machines
	passing r_int64() fits into a regular integer.

	- add test_translation.py, which finds quickly the annotation error
	mentioned above, and no other, so there are chances that it works
	now.

diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -277,7 +277,7 @@
                     _setinfo(space, w_info, "GetSystemTimeAsFileTime()",
                              time_increment[0] * 1e-7, False, True)
             if return_ns:
-                return space.newlong_from_rarith_int(r_int64(microseconds) * 10**3)
+                return space.newint(r_int64(microseconds) * 10**3)
             else:
                 tv_sec = microseconds / 10**6
                 tv_usec = microseconds % 10**6
@@ -302,7 +302,7 @@
                     if w_info is not None:
                         _setinfo(space, w_info, "gettimeofday()", 1e-6, False, True)
                     if return_ns:
-                        return space.newlong_from_rarith_int(
+                        return space.newint(
                             r_int64(timeval.c_tv_sec) * 10**9 +
                             r_int64(timeval.c_tv_usec) * 10**3)
                     else:
@@ -315,9 +315,9 @@
                 if w_info is not None:
                     _setinfo(space, w_info, "ftime()", 1e-3, False, True)
                 if return_ns:
-                    return space.newlong_from_rarith_int(
+                    return space.newint(
                         r_int64(t.c_time) * 10**9 +
-                        r_int64(t.c_millitm) * 10**6)
+                        r_int64(intmask(t.c_millitm)) * 10**6)
                 else:
                     return space.newfloat(
                         widen(t.c_time) +
@@ -327,7 +327,7 @@
                 _setinfo(space, w_info, "time()", 1.0, False, True)
             result = c_time(lltype.nullptr(rffi.TIME_TP.TO))
             if return_ns:
-                return space.newlong_from_rarith_int(r_int64(result) * 10**9)
+                return space.newint(r_int64(result) * 10**9)
             else:
                 return space.newfloat(float(result))
     
@@ -697,7 +697,7 @@
                         _setinfo(space, w_info, "clock_gettime(CLOCK_REALTIME)",
                                  res, False, True)
                 if return_ns:
-                    return space.newlong_from_rarith_int(_timespec_to_nanoseconds(timespec))
+                    return space.newint(_timespec_to_nanoseconds(timespec))
                 else:
                     return space.newfloat(_timespec_to_seconds(timespec))
     return _gettimeofday_impl(space, w_info, return_ns)
@@ -829,7 +829,7 @@
             if ret != 0:
                 raise exception_from_saved_errno(space, space.w_OSError)
             if return_ns:
-                return space.newlong_from_rarith_int(_timespec_to_nanoseconds(timespec))
+                return space.newint(_timespec_to_nanoseconds(timespec))
             else:
                 return space.newfloat(_timespec_to_seconds(timespec))
 
@@ -993,7 +993,7 @@
                 _setinfo(space, w_info, implementation, resolution, True, False)
 
             if return_ns:
-                return space.newlong_from_rarith_int(r_int64(tick_count) * 10**6)
+                return space.newint(r_int64(tick_count) * 10**6)
             else:
                 return space.newfloat(tick_count * 1e-3)
 
@@ -1017,7 +1017,7 @@
                 res = (numer / denom) * 1e-9
                 _setinfo(space, w_info, "mach_absolute_time()", res, True, False)
             if return_ns:
-                return space.newlong_from_rarith_int(nanosecs)
+                return space.newint(nanosecs)
             else:
                 secs = nanosecs / 10**9
                 rest = nanosecs % 10**9
@@ -1084,7 +1084,7 @@
                 _setinfo(space, w_info, "QueryPerformanceCounter()", resolution,
                          True, False)
             if return_ns:
-                return space.newlong_from_rarith_int(r_int64(diff) * 10**9 // time_state.divisor)
+                return space.newint(r_int64(diff) * 10**9 // time_state.divisor)
             else:
                 return space.newfloat(float(diff) / float(time_state.divisor))
 
@@ -1139,7 +1139,7 @@
         if w_info is not None:
             _setinfo(space, w_info, "GetProcessTimes()", 1e-7, True, False)
         if return_ns:
-            return space.newlong_from_rarith_int((r_int64(kernel_time2) + r_int64(user_time2)) * 10**2)
+            return space.newint((r_int64(kernel_time2) + r_int64(user_time2)) * 10**2)
         else:
             return space.newfloat((float(kernel_time2) + float(user_time2)) * 1e-7)
 else:
@@ -1168,7 +1168,7 @@
                         _setinfo(space, w_info,
                                  implementation, res, True, False)
                     if return_ns:
-                        return space.newlong_from_rarith_int(_timespec_to_nanoseconds(timespec))
+                        return space.newint(_timespec_to_nanoseconds(timespec))
                     else:
                         return space.newfloat(_timespec_to_seconds(timespec))
 
@@ -1182,7 +1182,7 @@
                         _setinfo(space, w_info,
                                  "getrusage(RUSAGE_SELF)", 1e-6, True, False)
                     if return_ns:
-                        return space.newlong_from_rarith_int(
+                        return space.newint(
                             decode_timeval_ns(rusage.c_ru_utime) +
                             decode_timeval_ns(rusage.c_ru_stime))
                     else:
@@ -1199,7 +1199,7 @@
                                  1.0 / rposix.CLOCK_TICKS_PER_SECOND,
                                  True, False)
                     if return_ns:
-                        return space.newlong_from_rarith_int(r_int64(cpu_time) * 10**9 // int(rposix.CLOCK_TICKS_PER_SECOND))
+                        return space.newint(r_int64(cpu_time) * 10**9 // int(rposix.CLOCK_TICKS_PER_SECOND))
                     else:
                         return space.newfloat(float(cpu_time) / rposix.CLOCK_TICKS_PER_SECOND)
         return _clock_impl(space, w_info, return_ns)
@@ -1233,7 +1233,7 @@
         _setinfo(space, w_info,
                  "clock()", 1.0 / CLOCKS_PER_SEC, True, False)
     if return_ns:
-        return space.newlong_from_rarith_int(r_int64(value) * 10**9 // CLOCKS_PER_SEC)
+        return space.newint(r_int64(value) * 10**9 // CLOCKS_PER_SEC)
     else:
         return space.newfloat(float(value) / CLOCKS_PER_SEC)
 
diff --git a/pypy/module/time/test/test_ztranslation.py b/pypy/module/time/test/test_ztranslation.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/time/test/test_ztranslation.py
@@ -0,0 +1,4 @@
+from pypy.objspace.fake.checkmodule import checkmodule
+
+def test_checkmodule():
+    checkmodule('time')

From pypy.commits at gmail.com  Fri Nov  8 02:37:44 2019
From: pypy.commits at gmail.com (CFSworks)
Date: Thu, 07 Nov 2019 23:37:44 -0800 (PST)
Subject: [pypy-commit] pypy default: cpyext: Respect tp_dict on PyType_Ready
Message-ID: <5dc51b48.1c69fb81.bc2a.aa58@mx.google.com>

Author: Sam Edwards <CFSworks at gmail.com>
Branch: 
Changeset: r97994:c5b62ebd9ccd
Date: 2019-11-06 16:51 -0700
http://bitbucket.org/pypy/pypy/changeset/c5b62ebd9ccd/

Log:	cpyext: Respect tp_dict on PyType_Ready

diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -452,6 +452,30 @@
         assert module.hack_tp_dict(obj, "b") == 2
 
 
+    def test_tp_dict_ready(self):
+        module = self.import_extension('foo', [
+           ("new_obj", "METH_NOARGS",
+            '''
+                PyObject *obj;
+                obj = PyObject_New(PyObject, &Foo_Type);
+                return obj;
+            '''
+            )], prologue='''
+            static PyTypeObject Foo_Type = {
+                PyVarObject_HEAD_INIT(NULL, 0)
+                "foo.foo",
+            };
+            ''', more_init = '''
+                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
+                Foo_Type.tp_dict = PyDict_New();
+                PyDict_SetItemString(Foo_Type.tp_dict, "inserted", Py_True);
+                if (PyType_Ready(&Foo_Type) < 0) INITERROR;
+            ''')
+
+        obj = module.new_obj()
+        assert type(obj).inserted is True
+
+
     def test_tp_descr_get(self):
         module = self.import_extension('foo', [
            ("tp_descr_get", "METH_O",
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -538,6 +538,13 @@
         convert_getset_defs(space, dict_w, pto.c_tp_getset, self)
         convert_member_defs(space, dict_w, pto.c_tp_members, self)
 
+        w_dict = from_ref(space, pto.c_tp_dict)
+        if w_dict is not None:
+            dictkeys_w = space.listview(w_dict)
+            for w_key in dictkeys_w:
+                key = space.text_w(w_key)
+                dict_w[key] = space.getitem(w_dict, w_key)
+
         name = rffi.charp2str(cts.cast('char*', pto.c_tp_name))
         flag_heaptype = pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE
         if flag_heaptype:

From pypy.commits at gmail.com  Fri Nov  8 14:20:43 2019
From: pypy.commits at gmail.com (arigo)
Date: Fri, 08 Nov 2019 11:20:43 -0800 (PST)
Subject: [pypy-commit] pypy py3.7-pep564: Close branch py3.7-pep564
Message-ID: <5dc5c00b.1c69fb81.29693.81fe@mx.google.com>

Author: Armin Rigo <armin.rigo at gmail.com>
Branch: py3.7-pep564
Changeset: r97995:7e766ef19d64
Date: 2019-11-08 19:20 +0000
http://bitbucket.org/pypy/pypy/changeset/7e766ef19d64/

Log:	Close branch py3.7-pep564


From pypy.commits at gmail.com  Fri Nov  8 14:21:02 2019
From: pypy.commits at gmail.com (arigo)
Date: Fri, 08 Nov 2019 11:21:02 -0800 (PST)
Subject: [pypy-commit] pypy py3.7: Merged in py3.7-pep564 (pull request #677)
Message-ID: <5dc5c01e.1c69fb81.3141e.d9c0@mx.google.com>

Author: Armin Rigo <armin.rigo at gmail.com>
Branch: py3.7
Changeset: r97996:d70821cfb56e
Date: 2019-11-08 19:20 +0000
http://bitbucket.org/pypy/pypy/changeset/d70821cfb56e/

Log:	Merged in py3.7-pep564 (pull request #677)

	PEP 564 implementation

diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -10,7 +10,7 @@
     str_decode_locale_surrogateescape, unicode_encode_locale_surrogateescape)
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rlib.rarithmetic import (
-    intmask, r_ulonglong, r_longfloat, widen, ovfcheck, ovfcheck_float_to_int)
+    intmask, r_ulonglong, r_longfloat, r_int64, widen, ovfcheck, ovfcheck_float_to_int)
 from rpython.rlib.rtime import (GETTIMEOFDAY_NO_TZ, TIMEVAL,
                                 HAVE_GETTIMEOFDAY, HAVE_FTIME)
 from rpython.rlib import rposix, rtime
@@ -132,7 +132,7 @@
         def __init__(self):
             self.n_overflow = 0
             self.last_ticks = 0
-            self.divisor = 0.0
+            self.divisor = 0
             self.counter_start = 0
 
         def check_GetTickCount64(self, *args):
@@ -252,7 +252,7 @@
                                             'GetSystemTimeAdjustment',
                                             [LPDWORD, LPDWORD, rwin32.LPBOOL],
                                             rffi.INT)
-    def gettimeofday(space, w_info=None):
+    def _gettimeofday_impl(space, w_info, return_ns):
         with lltype.scoped_alloc(rwin32.FILETIME) as system_time:
             _GetSystemTimeAsFileTime(system_time)
             quad_part = (system_time.c_dwLowDateTime |
@@ -267,8 +267,6 @@
             offset = (r_ulonglong(16384) * r_ulonglong(27) * r_ulonglong(390625)
                      * r_ulonglong(79) * r_ulonglong(853))
             microseconds = quad_part / 10 - offset
-            tv_sec = microseconds / 1000000
-            tv_usec = microseconds % 1000000
             if w_info:
                 with lltype.scoped_alloc(LPDWORD.TO, 1) as time_adjustment, \
                      lltype.scoped_alloc(LPDWORD.TO, 1) as time_increment, \
@@ -278,7 +276,12 @@
 
                     _setinfo(space, w_info, "GetSystemTimeAsFileTime()",
                              time_increment[0] * 1e-7, False, True)
-            return space.newfloat(tv_sec + tv_usec * 1e-6)
+            if return_ns:
+                return space.newint(r_int64(microseconds) * 10**3)
+            else:
+                tv_sec = microseconds / 10**6
+                tv_usec = microseconds % 10**6
+                return space.newfloat(tv_sec + tv_usec * 1e-6)
 else:
     if HAVE_GETTIMEOFDAY:
         if GETTIMEOFDAY_NO_TZ:
@@ -287,7 +290,7 @@
         else:
             c_gettimeofday = external('gettimeofday',
                                       [lltype.Ptr(TIMEVAL), rffi.VOIDP], rffi.INT)
-    def gettimeofday(space, w_info=None):
+    def _gettimeofday_impl(space, w_info, return_ns):
         if HAVE_GETTIMEOFDAY:
             with lltype.scoped_alloc(TIMEVAL) as timeval:
                 if GETTIMEOFDAY_NO_TZ:
@@ -298,22 +301,38 @@
                 if rffi.cast(rffi.LONG, errcode) == 0:
                     if w_info is not None:
                         _setinfo(space, w_info, "gettimeofday()", 1e-6, False, True)
-                    return space.newfloat(
-                        widen(timeval.c_tv_sec) +
-                        widen(timeval.c_tv_usec) * 1e-6)
+                    if return_ns:
+                        return space.newint(
+                            r_int64(timeval.c_tv_sec) * 10**9 +
+                            r_int64(timeval.c_tv_usec) * 10**3)
+                    else:
+                        return space.newfloat(
+                            widen(timeval.c_tv_sec) +
+                            widen(timeval.c_tv_usec) * 1e-6)
         if HAVE_FTIME:
             with lltype.scoped_alloc(TIMEB) as t:
                 c_ftime(t)
-                result = (widen(t.c_time) +
-                          widen(t.c_millitm) * 0.001)
                 if w_info is not None:
-                    _setinfo(space, w_info, "ftime()", 1e-3,
-                             False, True)
-            return space.newfloat(result)
+                    _setinfo(space, w_info, "ftime()", 1e-3, False, True)
+                if return_ns:
+                    return space.newint(
+                        r_int64(t.c_time) * 10**9 +
+                        r_int64(intmask(t.c_millitm)) * 10**6)
+                else:
+                    return space.newfloat(
+                        widen(t.c_time) +
+                        widen(t.c_millitm) * 1e-3)
         else:
             if w_info:
                 _setinfo(space, w_info, "time()", 1.0, False, True)
-            return space.newint(c_time(lltype.nullptr(rffi.TIME_TP.TO)))
+            result = c_time(lltype.nullptr(rffi.TIME_TP.TO))
+            if return_ns:
+                return space.newint(r_int64(result) * 10**9)
+            else:
+                return space.newfloat(float(result))
+    
+    def gettimeofday(space, w_info=None):
+        return _gettimeofday_impl(space, w_info, False)
 
 TM_P = lltype.Ptr(tm)
 c_time = external('time', [rffi.TIME_TP], rffi.TIME_T)
@@ -663,11 +682,7 @@
     if not 0 <= rffi.getintfield(t_ref, 'c_tm_yday') <= 365:
         raise oefmt(space.w_ValueError, "day of year out of range")
 
-def time(space, w_info=None):
-    """time() -> floating point number
-
-    Return the current time in seconds since the Epoch.
-    Fractions of a second may be present if the system clock provides them."""
+def _time_impl(space, w_info, return_ns):
     if HAS_CLOCK_GETTIME:
         with lltype.scoped_alloc(TIMESPEC) as timespec:
             ret = c_clock_gettime(rtime.CLOCK_REALTIME, timespec)
@@ -681,8 +696,24 @@
                             res = 1e-9
                         _setinfo(space, w_info, "clock_gettime(CLOCK_REALTIME)",
                                  res, False, True)
-                return space.newfloat(_timespec_to_seconds(timespec))
-    return gettimeofday(space, w_info)
+                if return_ns:
+                    return space.newint(_timespec_to_nanoseconds(timespec))
+                else:
+                    return space.newfloat(_timespec_to_seconds(timespec))
+    return _gettimeofday_impl(space, w_info, return_ns)
+
+def time(space, w_info=None):
+    """time() -> floating point number
+
+    Return the current time in seconds since the Epoch.
+    Fractions of a second may be present if the system clock provides them."""
+    return _time_impl(space, w_info, False)
+
+def time_ns(space, w_info=None):
+    """time_ns() -> int
+    
+    Return the current time in nanoseconds since the Epoch."""
+    return _time_impl(space, w_info, True)
 
 def ctime(space, w_seconds=None):
     """ctime([seconds]) -> string
@@ -789,17 +820,38 @@
     def _timespec_to_seconds(timespec):
         return widen(timespec.c_tv_sec) + widen(timespec.c_tv_nsec) * 1e-9
 
-    @unwrap_spec(clk_id='c_int')
-    def clock_gettime(space, clk_id):
+    def _timespec_to_nanoseconds(timespec):
+        return r_int64(timespec.c_tv_sec) * 10**9 + r_int64(timespec.c_tv_nsec)
+    
+    def _clock_gettime_impl(space, clk_id, return_ns):
         with lltype.scoped_alloc(TIMESPEC) as timespec:
             ret = c_clock_gettime(clk_id, timespec)
             if ret != 0:
                 raise exception_from_saved_errno(space, space.w_OSError)
-            secs = _timespec_to_seconds(timespec)
-        return space.newfloat(secs)
+            if return_ns:
+                return space.newint(_timespec_to_nanoseconds(timespec))
+            else:
+                return space.newfloat(_timespec_to_seconds(timespec))
+
+    @unwrap_spec(clk_id='c_int')
+    def clock_gettime(space, clk_id):
+        """clock_gettime(clk_id) -> float
+    
+        Return the time of the specified clock clk_id."""
+        return _clock_gettime_impl(space, clk_id, False)
+
+    @unwrap_spec(clk_id='c_int')
+    def clock_gettime_ns(space, clk_id):
+        """clock_gettime_ns(clk_id) -> int
+    
+        Return the time of the specified clock clk_id as nanoseconds."""
+        return _clock_gettime_impl(space, clk_id, True)
 
     @unwrap_spec(clk_id='c_int', secs=float)
     def clock_settime(space, clk_id, secs):
+        """clock_settime(clk_id, time)
+    
+        Set the time of the specified clock clk_id."""
         with lltype.scoped_alloc(TIMESPEC) as timespec:
             integer_secs = rffi.cast(TIMESPEC.c_tv_sec, secs)
             frac = secs - widen(integer_secs)
@@ -809,8 +861,23 @@
             if ret != 0:
                 raise exception_from_saved_errno(space, space.w_OSError)
 
+    @unwrap_spec(clk_id='c_int', ns=r_int64)
+    def clock_settime_ns(space, clk_id, ns):
+        """clock_settime_ns(clk_id, time)
+    
+        Set the time of the specified clock clk_id with nanoseconds."""
+        with lltype.scoped_alloc(TIMESPEC) as timespec:
+            rffi.setintfield(timespec, 'c_tv_sec', ns // 10**9)
+            rffi.setintfield(timespec, 'c_tv_nsec', ns % 10**9)
+            ret = c_clock_settime(clk_id, timespec)
+            if ret != 0:
+                raise exception_from_saved_errno(space, space.w_OSError)
+
     @unwrap_spec(clk_id='c_int')
     def clock_getres(space, clk_id):
+        """clock_getres(clk_id) -> floating point number
+    
+        Return the resolution (precision) of the specified clock clk_id."""
         with lltype.scoped_alloc(TIMESPEC) as timespec:
             ret = c_clock_getres(clk_id, timespec)
             if ret != 0:
@@ -893,19 +960,18 @@
 if HAS_MONOTONIC:
     if _WIN:
         _GetTickCount = rwin32.winexternal('GetTickCount', [], rwin32.DWORD)
-        def monotonic(space, w_info=None):
+        def _monotonic_impl(space, w_info, return_ns):
             result = 0
             HAS_GETTICKCOUNT64 = time_state.check_GetTickCount64()
             if HAS_GETTICKCOUNT64:
-                result = time_state.GetTickCount64() * 1e-3
+                tick_count = time_state.GetTickCount64()
             else:
                 ticks = _GetTickCount()
                 if ticks < time_state.last_ticks:
                     time_state.n_overflow += 1
                 time_state.last_ticks = ticks
-                result = math.ldexp(time_state.n_overflow, 32)
-                result = result + ticks
-                result = result * 1e-3
+                tick_count = math.ldexp(time_state.n_overflow, 32)
+                tick_count = tick_count + ticks
 
             if w_info is not None:
                 if HAS_GETTICKCOUNT64:
@@ -925,7 +991,11 @@
                             rwin32.lastSavedWindowsError("GetSystemTimeAdjustment"))
                     resolution = resolution * time_increment[0]
                 _setinfo(space, w_info, implementation, resolution, True, False)
-            return space.newfloat(result)
+
+            if return_ns:
+                return space.newint(r_int64(tick_count) * 10**6)
+            else:
+                return space.newfloat(tick_count * 1e-3)
 
     elif _MACOSX:
         c_mach_timebase_info = external('mach_timebase_info',
@@ -936,30 +1006,33 @@
         timebase_info = lltype.malloc(cConfig.TIMEBASE_INFO, flavor='raw',
                                       zero=True, immortal=True)
 
-        def monotonic(space, w_info=None):
+        def _monotonic_impl(space, w_info, return_ns):
             if rffi.getintfield(timebase_info, 'c_denom') == 0:
                 c_mach_timebase_info(timebase_info)
             time = rffi.cast(lltype.Signed, c_mach_absolute_time())
             numer = rffi.getintfield(timebase_info, 'c_numer')
             denom = rffi.getintfield(timebase_info, 'c_denom')
-            nanosecs = time * numer / denom
+            nanosecs = r_int64(time) * numer / denom
             if w_info is not None:
                 res = (numer / denom) * 1e-9
                 _setinfo(space, w_info, "mach_absolute_time()", res, True, False)
-            secs = nanosecs / 10**9
-            rest = nanosecs % 10**9
-            return space.newfloat(float(secs) + float(rest) * 1e-9)
+            if return_ns:
+                return space.newint(nanosecs)
+            else:
+                secs = nanosecs / 10**9
+                rest = nanosecs % 10**9
+                return space.newfloat(float(secs) + float(rest) * 1e-9)
 
     else:
         assert _POSIX
-        def monotonic(space, w_info=None):
+        def _monotonic_impl(space, w_info, return_ns):
             if rtime.CLOCK_HIGHRES is not None:
                 clk_id = rtime.CLOCK_HIGHRES
                 implementation = "clock_gettime(CLOCK_HIGHRES)"
             else:
                 clk_id = rtime.CLOCK_MONOTONIC
                 implementation = "clock_gettime(CLOCK_MONOTONIC)"
-            w_result = clock_gettime(space, clk_id)
+            w_result = _clock_gettime_impl(space, clk_id, return_ns)
             if w_info is not None:
                 with lltype.scoped_alloc(TIMESPEC) as tsres:
                     ret = c_clock_getres(clk_id, tsres)
@@ -970,6 +1043,18 @@
                 _setinfo(space, w_info, implementation, res, True, False)
             return w_result
 
+    def monotonic(space, w_info=None):
+        """monotonic() -> float
+    
+        Monotonic clock, cannot go backward."""
+        return _monotonic_impl(space, w_info, False)
+
+    def monotonic_ns(space, w_info=None):
+        """monotonic_ns() -> int
+    
+        Monotonic clock, cannot go backward, as nanoseconds."""
+        return _monotonic_impl(space, w_info, True)
+
 if _WIN:
     # hacking to avoid LARGE_INTEGER which is a union...
     QueryPerformanceCounter = external(
@@ -981,46 +1066,61 @@
     QueryPerformanceFrequency = rwin32.winexternal(
         'QueryPerformanceFrequency', [rffi.CArrayPtr(lltype.SignedLongLong)],
         rffi.INT)
-    def win_perf_counter(space, w_info=None):
+    def _win_perf_counter_impl(space, w_info, return_ns):
         with lltype.scoped_alloc(rffi.CArray(rffi.lltype.SignedLongLong), 1) as a:
             succeeded = True
-            if time_state.divisor == 0.0:
+            if time_state.divisor == 0:
                 QueryPerformanceCounter(a)
                 time_state.counter_start = a[0]
                 succeeded = QueryPerformanceFrequency(a)
-                time_state.divisor = float(a[0])
-            if succeeded and time_state.divisor != 0.0:
+                time_state.divisor = a[0]
+            if succeeded and time_state.divisor != 0:
                 QueryPerformanceCounter(a)
                 diff = a[0] - time_state.counter_start
             else:
                 raise ValueError("Failed to generate the result.")
-            resolution = 1 / time_state.divisor
             if w_info is not None:
+                resolution = 1 / float(time_state.divisor)
                 _setinfo(space, w_info, "QueryPerformanceCounter()", resolution,
                          True, False)
-            return space.newfloat(float(diff) / time_state.divisor)
+            if return_ns:
+                return space.newint(r_int64(diff) * 10**9 // time_state.divisor)
+            else:
+                return space.newfloat(float(diff) / float(time_state.divisor))
 
-    def perf_counter(space, w_info=None):
+    def _perf_counter_impl(space, w_info, return_ns):
         try:
-            return win_perf_counter(space, w_info=w_info)
+            return _win_perf_counter_impl(space, w_info, return_ns)
         except ValueError:
             if HAS_MONOTONIC:
                 try:
-                    return monotonic(space, w_info=w_info)
+                    return _monotonic_impl(space, w_info, return_ns)
                 except Exception:
                     pass
-        return time(space, w_info=w_info)
+        return _time_impl(space, w_info, return_ns)
 else:
-    def perf_counter(space, w_info=None):
+    def _perf_counter_impl(space, w_info, return_ns):
         if HAS_MONOTONIC:
             try:
-                return monotonic(space, w_info=w_info)
+                return _monotonic_impl(space, w_info, return_ns)
             except Exception:
                 pass
-        return time(space, w_info=w_info)
+        return _time_impl(space, w_info, return_ns)
+
+def perf_counter(space, w_info=None):
+    """perf_counter() -> float
+    
+    Performance counter for benchmarking."""
+    return _perf_counter_impl(space, w_info, False)
+
+def perf_counter_ns(space, w_info=None):
+    """perf_counter_ns() -> int
+    
+    Performance counter for benchmarking as nanoseconds."""
+    return _perf_counter_impl(space, w_info, True)
 
 if _WIN:
-    def process_time(space, w_info=None):
+    def _process_time_impl(space, w_info, return_ns):
         from rpython.rlib.rposix import GetCurrentProcess, GetProcessTimes
         current_process = GetCurrentProcess()
         with lltype.scoped_alloc(rwin32.FILETIME) as creation_time, \
@@ -1038,11 +1138,14 @@
                           r_ulonglong(user_time.c_dwHighDateTime) << 32)
         if w_info is not None:
             _setinfo(space, w_info, "GetProcessTimes()", 1e-7, True, False)
-        return space.newfloat((float(kernel_time2) + float(user_time2)) * 1e-7)
+        if return_ns:
+            return space.newint((r_int64(kernel_time2) + r_int64(user_time2)) * 10**2)
+        else:
+            return space.newfloat((float(kernel_time2) + float(user_time2)) * 1e-7)
 else:
     have_times = hasattr(rposix, 'c_times')
 
-    def process_time(space, w_info=None):
+    def _process_time_impl(space, w_info, return_ns):
         if HAS_CLOCK_GETTIME and (
                 rtime.CLOCK_PROF is not None or
                 rtime.CLOCK_PROCESS_CPUTIME_ID is not None):
@@ -1064,44 +1167,61 @@
                                 res = 1e-9
                         _setinfo(space, w_info,
                                  implementation, res, True, False)
-                    return space.newfloat(_timespec_to_seconds(timespec))
+                    if return_ns:
+                        return space.newint(_timespec_to_nanoseconds(timespec))
+                    else:
+                        return space.newfloat(_timespec_to_seconds(timespec))
 
         if True: # XXX available except if it isn't?
             from rpython.rlib.rtime import (c_getrusage, RUSAGE, RUSAGE_SELF,
-                                            decode_timeval)
+                                            decode_timeval, decode_timeval_ns)
             with lltype.scoped_alloc(RUSAGE) as rusage:
                 ret = c_getrusage(RUSAGE_SELF, rusage)
                 if ret == 0:
                     if w_info is not None:
                         _setinfo(space, w_info,
                                  "getrusage(RUSAGE_SELF)", 1e-6, True, False)
-                    return space.newfloat(decode_timeval(rusage.c_ru_utime) +
-                                          decode_timeval(rusage.c_ru_stime))
+                    if return_ns:
+                        return space.newint(
+                            decode_timeval_ns(rusage.c_ru_utime) +
+                            decode_timeval_ns(rusage.c_ru_stime))
+                    else:
+                        return space.newfloat(decode_timeval(rusage.c_ru_utime) +
+                                              decode_timeval(rusage.c_ru_stime))
         if have_times:
             with lltype.scoped_alloc(rposix.TMS) as tms:
                 ret = rposix.c_times(tms)
                 if rffi.cast(lltype.Signed, ret) != -1:
-                    cpu_time = float(rffi.cast(lltype.Signed,
-                                               tms.c_tms_utime) +
-                                     rffi.cast(lltype.Signed,
-                                               tms.c_tms_stime))
+                    cpu_time = (rffi.cast(lltype.Signed, tms.c_tms_utime) +
+                                rffi.cast(lltype.Signed, tms.c_tms_stime))
                     if w_info is not None:
                         _setinfo(space, w_info, "times()",
                                  1.0 / rposix.CLOCK_TICKS_PER_SECOND,
                                  True, False)
-                    return space.newfloat(cpu_time / rposix.CLOCK_TICKS_PER_SECOND)
-        return clock(space)
+                    if return_ns:
+                        return space.newint(r_int64(cpu_time) * 10**9 // int(rposix.CLOCK_TICKS_PER_SECOND))
+                    else:
+                        return space.newfloat(float(cpu_time) / rposix.CLOCK_TICKS_PER_SECOND)
+        return _clock_impl(space, w_info, return_ns)
+
+def process_time(space, w_info=None):
+    """process_time() -> float
+
+    Process time for profiling: sum of the kernel and user-space CPU time."""
+    return _process_time_impl(space, w_info, False)
+
+def process_time_ns(space, w_info=None):
+    """process_time() -> int
+
+    Process time for profiling as nanoseconds:
+    sum of the kernel and user-space CPU time"""
+    return _process_time_impl(space, w_info, True)
 
 _clock = external('clock', [], rposix.CLOCK_T)
-def clock(space, w_info=None):
-    """clock() -> floating point number
-
-    Return the CPU time or real time since the start of the process or since
-    the first call to clock().  This has as much precision as the system
-    records."""
+def _clock_impl(space, w_info, return_ns):
     if _WIN:
         try:
-            return win_perf_counter(space, w_info=w_info)
+            return _win_perf_counter_impl(space, w_info, return_ns)
         except ValueError:
             pass
     value = widen(_clock())
@@ -1112,7 +1232,18 @@
     if w_info is not None:
         _setinfo(space, w_info,
                  "clock()", 1.0 / CLOCKS_PER_SEC, True, False)
-    return space.newfloat(float(value) / CLOCKS_PER_SEC)
+    if return_ns:
+        return space.newint(r_int64(value) * 10**9 // CLOCKS_PER_SEC)
+    else:
+        return space.newfloat(float(value) / CLOCKS_PER_SEC)
+
+def clock(space, w_info=None):
+    """clock() -> floating point number
+
+    Return the CPU time or real time since the start of the process or since
+    the first call to clock().  This has as much precision as the system
+    records."""
+    return _clock_impl(space, w_info, False)
 
 
 def _setinfo(space, w_info, impl, res, mono, adj):
diff --git a/pypy/module/time/moduledef.py b/pypy/module/time/moduledef.py
--- a/pypy/module/time/moduledef.py
+++ b/pypy/module/time/moduledef.py
@@ -11,6 +11,7 @@
 
     interpleveldefs = {
         'time': 'interp_time.time',
+        'time_ns': 'interp_time.time_ns',
         'clock': 'interp_time.clock',
         'ctime': 'interp_time.ctime',
         'asctime': 'interp_time.asctime',
@@ -21,18 +22,23 @@
         'sleep' : 'interp_time.sleep',
         '_STRUCT_TM_ITEMS': 'space.wrap(interp_time._STRUCT_TM_ITEMS)',
         'perf_counter': 'interp_time.perf_counter',
+        'perf_counter_ns': 'interp_time.perf_counter_ns',
         'process_time': 'interp_time.process_time',
+        'process_time_ns': 'interp_time.process_time_ns',
     }
 
     if rtime.HAS_CLOCK_GETTIME:
         interpleveldefs['clock_gettime'] = 'interp_time.clock_gettime'
+        interpleveldefs['clock_gettime_ns'] = 'interp_time.clock_gettime_ns'
         interpleveldefs['clock_settime'] = 'interp_time.clock_settime'
+        interpleveldefs['clock_settime_ns'] = 'interp_time.clock_settime_ns'
         interpleveldefs['clock_getres'] = 'interp_time.clock_getres'
         for constant in rtime.ALL_DEFINED_CLOCKS:
             interpleveldefs[constant] = 'space.wrap(%d)' % (
                 getattr(rtime, constant),)
     if HAS_MONOTONIC:
         interpleveldefs['monotonic'] = 'interp_time.monotonic'
+        interpleveldefs['monotonic_ns'] = 'interp_time.monotonic_ns'
     if os.name == "posix":
         interpleveldefs['tzset'] = 'interp_time.tzset'
 
diff --git a/pypy/module/time/test/test_time.py b/pypy/module/time/test/test_time.py
--- a/pypy/module/time/test/test_time.py
+++ b/pypy/module/time/test/test_time.py
@@ -28,12 +28,22 @@
     def test_time(self):
         import time
         t1 = time.time()
-        assert isinstance(time.time(), float)
-        assert time.time() != 0.0 # 0.0 means failure
+        assert isinstance(t1, float)
+        assert t1 != 0.0  # 0.0 means failure
         time.sleep(0.02)
         t2 = time.time()
         assert t1 != t2       # the resolution should be at least 0.01 secs
 
+    def test_time_ns(self):
+        import time
+        t1 = time.time_ns()
+        assert isinstance(t1, int)
+        assert t1 != 0  # 0 means failure
+        time.sleep(0.02)
+        t2 = time.time_ns()
+        assert t1 != t2       # the resolution should be at least 0.01 secs
+        assert abs(time.time() - time.time_ns() * 1e-9) < 0.1
+
     def test_clock_realtime(self):
         import time
         if not hasattr(time, 'clock_gettime'):
@@ -44,6 +54,18 @@
         t2 = time.clock_gettime(time.CLOCK_REALTIME)
         assert t1 != t2
 
+    def test_clock_realtime_ns(self):
+        import time
+        if not hasattr(time, 'clock_gettime_ns'):
+            skip("need time.clock_gettime_ns()")
+        t1 = time.clock_gettime_ns(time.CLOCK_REALTIME)
+        assert isinstance(t1, int)
+        time.sleep(time.clock_getres(time.CLOCK_REALTIME))
+        t2 = time.clock_gettime_ns(time.CLOCK_REALTIME)
+        assert t1 != t2
+        assert abs(time.clock_gettime(time.CLOCK_REALTIME) -
+                   time.clock_gettime_ns(time.CLOCK_REALTIME) * 1e-9) < 0.1
+
     def test_clock_monotonic(self):
         import time
         if not (hasattr(time, 'clock_gettime') and
@@ -55,6 +77,19 @@
         t2 = time.clock_gettime(time.CLOCK_MONOTONIC)
         assert t1 < t2
 
+    def test_clock_monotonic_ns(self):
+        import time
+        if not (hasattr(time, 'clock_gettime_ns') and
+                hasattr(time, 'CLOCK_MONOTONIC')):
+            skip("need time.clock_gettime()/CLOCK_MONOTONIC")
+        t1 = time.clock_gettime_ns(time.CLOCK_MONOTONIC)
+        assert isinstance(t1, int)
+        time.sleep(time.clock_getres(time.CLOCK_MONOTONIC))
+        t2 = time.clock_gettime_ns(time.CLOCK_MONOTONIC)
+        assert t1 < t2
+        assert abs(time.clock_gettime(time.CLOCK_MONOTONIC) -
+                   time.clock_gettime_ns(time.CLOCK_MONOTONIC) * 1e-9) < 0.1
+
     def test_ctime(self):
         import time
         raises(TypeError, time.ctime, "foo")
@@ -398,10 +433,24 @@
         t2 = time.monotonic()
         assert t1 < t2
 
+    def test_monotonic_ns(self):
+        import time
+        t1 = time.monotonic_ns()
+        assert isinstance(t1, int)
+        time.sleep(0.02)
+        t2 = time.monotonic_ns()
+        assert t1 < t2
+        assert abs(time.monotonic() - time.monotonic_ns() * 1e-9) < 0.1
+
     def test_perf_counter(self):
         import time
         assert isinstance(time.perf_counter(), float)
 
+    def test_perf_counter_ns(self):
+        import time
+        assert isinstance(time.perf_counter_ns(), int)
+        assert abs(time.perf_counter() - time.perf_counter_ns() * 1e-9) < 0.1
+
     def test_process_time(self):
         import time
         t1 = time.process_time()
@@ -411,6 +460,16 @@
         # process_time() should not include time spent during sleep
         assert (t2 - t1) < 0.05
 
+    def test_process_time_ns(self):
+        import time
+        t1 = time.process_time_ns()
+        assert isinstance(t1, int)
+        time.sleep(0.1)
+        t2 = time.process_time_ns()
+        # process_time_ns() should not include time spent during sleep
+        assert (t2 - t1) < 5 * 10**7
+        assert abs(time.process_time() - time.process_time_ns() * 1e-9) < 0.1
+
     def test_get_clock_info(self):
         import time
         clocks = ['clock', 'perf_counter', 'process_time', 'time']
diff --git a/pypy/module/time/test/test_ztranslation.py b/pypy/module/time/test/test_ztranslation.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/time/test/test_ztranslation.py
@@ -0,0 +1,4 @@
+from pypy.objspace.fake.checkmodule import checkmodule
+
+def test_checkmodule():
+    checkmodule('time')
diff --git a/rpython/rlib/rtime.py b/rpython/rlib/rtime.py
--- a/rpython/rlib/rtime.py
+++ b/rpython/rlib/rtime.py
@@ -9,7 +9,7 @@
 from rpython.rtyper.tool import rffi_platform
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib.objectmodel import register_replacement_for
-from rpython.rlib.rarithmetic import intmask, UINT_MAX
+from rpython.rlib.rarithmetic import intmask, r_int64, UINT_MAX
 from rpython.rlib import rposix
 
 _WIN32 = sys.platform.startswith('win')
@@ -94,6 +94,10 @@
     return (float(rffi.getintfield(t, 'c_tv_sec')) +
             float(rffi.getintfield(t, 'c_tv_usec')) * 0.000001)
 
+def decode_timeval_ns(t):
+    return (r_int64(rffi.getintfield(t, 'c_tv_sec')) * 10**9 +
+            r_int64(rffi.getintfield(t, 'c_tv_usec')) * 10**3)
+
 
 def external(name, args, result, compilation_info=eci, **kwds):
     return rffi.llexternal(name, args, result,

From pypy.commits at gmail.com  Fri Nov  8 21:53:13 2019
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 08 Nov 2019 18:53:13 -0800 (PST)
Subject: [pypy-commit] pypy py3.6: hg merge default
Message-ID: <5dc62a19.1c69fb81.ab414.a994@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6
Changeset: r98001:9661af74538e
Date: 2019-11-09 02:52 +0000
http://bitbucket.org/pypy/pypy/changeset/9661af74538e/

Log:	hg merge default

diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -427,6 +427,30 @@
         assert module.hack_tp_dict(obj, "b") == 2
 
 
+    def test_tp_dict_ready(self):
+        module = self.import_extension('foo', [
+           ("new_obj", "METH_NOARGS",
+            '''
+                PyObject *obj;
+                obj = PyObject_New(PyObject, &Foo_Type);
+                return obj;
+            '''
+            )], prologue='''
+            static PyTypeObject Foo_Type = {
+                PyVarObject_HEAD_INIT(NULL, 0)
+                "foo.foo",
+            };
+            ''', more_init = '''
+                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
+                Foo_Type.tp_dict = PyDict_New();
+                PyDict_SetItemString(Foo_Type.tp_dict, "inserted", Py_True);
+                if (PyType_Ready(&Foo_Type) < 0) INITERROR;
+            ''')
+
+        obj = module.new_obj()
+        assert type(obj).inserted is True
+
+
     def test_tp_descr_get(self):
         module = self.import_extension('foo', [
            ("tp_descr_get", "METH_O",
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -540,6 +540,13 @@
         convert_getset_defs(space, dict_w, pto.c_tp_getset, self)
         convert_member_defs(space, dict_w, pto.c_tp_members, self)
 
+        w_dict = from_ref(space, pto.c_tp_dict)
+        if w_dict is not None:
+            dictkeys_w = space.listview(w_dict)
+            for w_key in dictkeys_w:
+                key = space.text_w(w_key)
+                dict_w[key] = space.getitem(w_dict, w_key)
+
         flag_heaptype = pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE
         if flag_heaptype:
             minsize = rffi.sizeof(PyHeapTypeObject.TO)

From pypy.commits at gmail.com  Fri Nov  8 21:53:06 2019
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 08 Nov 2019 18:53:06 -0800 (PST)
Subject: [pypy-commit] pypy default: Convert test_exec to apptest
Message-ID: <5dc62a12.1c69fb81.14ae5.d5fe@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r97997:bcfb9bf0b28b
Date: 2019-11-09 02:00 +0000
http://bitbucket.org/pypy/pypy/changeset/bcfb9bf0b28b/

Log:	Convert test_exec to apptest

diff --git a/pypy/interpreter/test/test_exec.py b/pypy/interpreter/test/apptest_exec.py
copy from pypy/interpreter/test/test_exec.py
copy to pypy/interpreter/test/apptest_exec.py
--- a/pypy/interpreter/test/test_exec.py
+++ b/pypy/interpreter/test/apptest_exec.py
@@ -2,271 +2,254 @@
 
 New for PyPy - Could be incorporated into CPython regression tests.
 """
-from rpython.tool.udir import udir
+def test_string():
+    g = {}
+    l = {}
+    exec "a = 3" in g, l
+    assert l['a'] == 3
 
+def test_localfill():
+    g = {}
+    exec "a = 3" in g
+    assert g['a'] == 3
 
-def test_file(space):
-    fn = udir.join('test_exec_file')
-    fn.write('abc=1\ncba=2\n')
-    space.appexec([space.wrap(str(fn))], '''
-        (filename):
-            fo = open(filename, 'r')
-            g = {}
-            exec fo in g
-            assert 'abc' in g
-            assert 'cba' in g
-    ''')
+def test_builtinsupply():
+    g = {}
+    exec "pass" in g
+    assert g.has_key('__builtins__')
 
+def test_invalidglobal():
+    def f():
+        exec 'pass' in 1
+    raises(TypeError,f)
 
-class AppTestExecStmt:
-    def test_string(self):
-        g = {}
-        l = {}
-        exec "a = 3" in g, l
-        assert l['a'] == 3
+def test_invalidlocal():
+    def f():
+        exec 'pass' in {}, 2
+    raises(TypeError,f)
 
-    def test_localfill(self):
-        g = {}
-        exec "a = 3" in g
-        assert g['a'] == 3
+def test_codeobject():
+    co = compile("a = 3", '<string>', 'exec')
+    g = {}
+    l = {}
+    exec co in g, l
+    assert l['a'] == 3
 
-    def test_builtinsupply(self):
-        g = {}
-        exec "pass" in g
-        assert g.has_key('__builtins__')
+def test_implicit():
+    a = 4
+    exec "a = 3"
+    assert a == 3
 
-    def test_invalidglobal(self):
+def test_tuplelocals():
+    g = {}
+    l = {}
+    exec ("a = 3", g, l)
+    assert l['a'] == 3
+
+def test_tupleglobals():
+    g = {}
+    exec ("a = 3", g)
+    assert g['a'] == 3
+
+def test_exceptionfallthrough():
+    def f():
+        exec 'raise TypeError' in {}
+    raises(TypeError,f)
+
+def test_global_stmt():
+    g = {}
+    l = {}
+    co = compile("global a; a=5", '', 'exec')
+    #import dis
+    #dis.dis(co)
+    exec co in g, l
+    assert l == {}
+    assert g['a'] == 5
+
+def test_specialcase_free_load():
+    exec """if 1:
         def f():
-            exec 'pass' in 1
-        raises(TypeError,f)
+            exec 'a=3'
+            return a
+        x = f()\n"""
+    assert x == 3
 
-    def test_invalidlocal(self):
+def test_specialcase_free_load2():
+    exec """if 1:
+        def f(a):
+            exec 'a=3'
+            return a
+        x = f(4)\n"""
+    assert x == 3
+
+def test_specialcase_globals_and_exec():
+    d = {}
+    exec """if 1:
+        b = 2
+        c = 3
+        d = 4
+        def f(a):
+            global b
+            exec 'd=42 ; b=7'
+            return a,b,c,d
+        #import dis
+        #dis.dis(f)
+        res = f(3)\n""" in d
+    r = d['res']
+    assert r == (3,2,3,42)
+
+def test_nested_names_are_not_confused():
+    def get_nested_class():
+        method_and_var = "var"
+        class Test(object):
+            def method_and_var(self):
+                return "method"
+            def test(self):
+                return method_and_var
+            def actual_global(self):
+                return str("global")
+            def str(self):
+                return str(self)
+        return Test()
+    t = get_nested_class()
+    assert t.actual_global() == "global"
+    assert t.test() == 'var'
+    assert t.method_and_var() == 'method'
+
+def test_import_star_shadows_global():
+    d = {'platform' : 3}
+    exec """if 1:
         def f():
-            exec 'pass' in {}, 2
-        raises(TypeError,f)
+            from sys import *
+            return platform
+        res = f()\n""" in d
+    import sys
+    assert d['res'] == sys.platform
 
-    def test_codeobject(self):
-        co = compile("a = 3", '<string>', 'exec')
-        g = {}
-        l = {}
-        exec co in g, l
-        assert l['a'] == 3
+def test_import_global_takes_precendence():
+    d = {'platform' : 3}
+    exec """if 1:
+        def f():
+            global platform
+            from sys import *
+            return platform
+        res = f()\n""" in d
+    import sys
+    assert d['platform'] == 3
 
-    def test_implicit(self):
-        a = 4
-        exec "a = 3"
-        assert a == 3
+def test_exec_load_name():
+    d = {'x': 2}
+    exec """if 1:
+        def f():
+            save = x
+            exec "x=3"
+            return x,save
+    \n""" in d
+    res = d['f']()
+    assert res == (3, 2)
 
-    def test_tuplelocals(self):
-        g = {}
-        l = {}
-        exec ("a = 3", g, l)
-        assert l['a'] == 3
+def test_space_bug():
+    d = {}
+    exec "x=5 " in d
+    assert d['x'] == 5
 
-    def test_tupleglobals(self):
-        g = {}
-        exec ("a = 3", g)
-        assert g['a'] == 3
+def test_synerr():
+    def x():
+        exec "1 2"
+    raises(SyntaxError, x)
 
-    def test_exceptionfallthrough(self):
-        def f():
-            exec 'raise TypeError' in {}
-        raises(TypeError,f)
+def test_mapping_as_locals():
+    import sys
+    if sys.version_info < (2,5) or not hasattr(sys, 'pypy_objspaceclass'):
+        skip("need CPython 2.5 or PyPy for non-dictionaries in exec statements")
+    class M(object):
+        def __getitem__(self, key):
+            return key
+        def __setitem__(self, key, value):
+            self.result[key] = value
+        def setdefault(self, key, value):
+            assert key == '__builtins__'
+    m = M()
+    m.result = {}
+    exec "x=m" in {}, m
+    assert m.result == {'x': 'm'}
+    exec "y=n" in m   # NOTE: this doesn't work in CPython 2.4
+    assert m.result == {'x': 'm', 'y': 'n'}
 
-    def test_global_stmt(self):
-        g = {}
-        l = {}
-        co = compile("global a; a=5", '', 'exec')
-        #import dis
-        #dis.dis(co)
-        exec co in g, l
-        assert l == {}
-        assert g['a'] == 5
+def test_filename():
+    try:
+        exec "'unmatched_quote"
+    except SyntaxError as msg:
+        assert msg.filename == '<string>'
+    try:
+        eval("'unmatched_quote")
+    except SyntaxError as msg:
+        assert msg.filename == '<string>'
 
-    def test_specialcase_free_load(self):
-        exec """if 1:
-            def f():
-                exec 'a=3'
-                return a
-            x = f()\n"""
-        assert x == 3
+def test_exec_and_name_lookups():
+    ns = {}
+    exec """def f():
+        exec 'x=1' in locals()
+        return x""" in ns
 
-    def test_specialcase_free_load2(self):
-        exec """if 1:
-            def f(a):
-                exec 'a=3'
-                return a
-            x = f(4)\n"""
-        assert x == 3
+    f = ns['f']
 
-    def test_specialcase_globals_and_exec(self):
-        d = {}
-        exec """if 1:
-            b = 2
-            c = 3 
-            d = 4 
-            def f(a):
-                global b
-                exec 'd=42 ; b=7'
-                return a,b,c,d
-            #import dis
-            #dis.dis(f)
-            res = f(3)\n""" in d
-        r = d['res']
-        assert r == (3,2,3,42)
+    try:
+        res = f()
+    except NameError as e: # keep py.test from exploding confused
+        raise e
 
-    def test_nested_names_are_not_confused(self):
-        def get_nested_class():
-            method_and_var = "var"
-            class Test(object):
-                def method_and_var(self):
-                    return "method"
-                def test(self):
-                    return method_and_var
-                def actual_global(self):
-                    return str("global")
-                def str(self):
-                    return str(self)
-            return Test()
-        t = get_nested_class()
-        assert t.actual_global() == "global" 
-        assert t.test() == 'var'
-        assert t.method_and_var() == 'method'
+    assert res == 1
 
-    def test_import_star_shadows_global(self):
-        d = {'platform' : 3}
-        exec """if 1:
-            def f():
-                from sys import *
-                return platform
-            res = f()\n""" in d
-        import sys
-        assert d['res'] == sys.platform
+def test_exec_unicode():
+    # 's' is a string
+    s = "x = u'\xd0\xb9\xd1\x86\xd1\x83\xd0\xba\xd0\xb5\xd0\xbd'"
+    # 'u' is a unicode
+    u = s.decode('utf-8')
+    exec u
+    assert len(x) == 6
+    assert ord(x[0]) == 0x0439
+    assert ord(x[1]) == 0x0446
+    assert ord(x[2]) == 0x0443
+    assert ord(x[3]) == 0x043a
+    assert ord(x[4]) == 0x0435
+    assert ord(x[5]) == 0x043d
 
-    def test_import_global_takes_precendence(self):
-        d = {'platform' : 3}
-        exec """if 1:
-            def f():
-                global platform
-                from sys import *
-                return platform
-            res = f()\n""" in d
-        import sys
-        assert d['platform'] == 3
+def test_eval_unicode():
+    u = "u'%s'" % unichr(0x1234)
+    v = eval(u)
+    assert v == unichr(0x1234)
 
-    def test_exec_load_name(self):
-        d = {'x': 2}
-        exec """if 1:
-            def f():
-                save = x 
-                exec "x=3"
-                return x,save
-        \n""" in d
-        res = d['f']()
-        assert res == (3, 2)
+def test_compile_unicode():
+    s = "x = u'\xd0\xb9\xd1\x86\xd1\x83\xd0\xba\xd0\xb5\xd0\xbd'"
+    u = s.decode('utf-8')
+    c = compile(u, '<input>', 'exec')
+    exec c
+    assert len(x) == 6
+    assert ord(x[0]) == 0x0439
 
-    def test_space_bug(self):
-        d = {}
-        exec "x=5 " in d
-        assert d['x'] == 5
-
-    def test_synerr(self):
-        def x():
-            exec "1 2"
-        raises(SyntaxError, x)
-
-    def test_mapping_as_locals(self):
-        import sys
-        if sys.version_info < (2,5) or not hasattr(sys, 'pypy_objspaceclass'):
-            skip("need CPython 2.5 or PyPy for non-dictionaries in exec statements")
-        class M(object):
-            def __getitem__(self, key):
-                return key
-            def __setitem__(self, key, value):
-                self.result[key] = value
-            def setdefault(self, key, value):
-                assert key == '__builtins__'
-        m = M()
-        m.result = {}
-        exec "x=m" in {}, m
-        assert m.result == {'x': 'm'}
-        exec "y=n" in m   # NOTE: this doesn't work in CPython 2.4
-        assert m.result == {'x': 'm', 'y': 'n'}
-
-    def test_filename(self):
-        try:
-            exec "'unmatched_quote"
-        except SyntaxError as msg:
-            assert msg.filename == '<string>'
-        try:
-            eval("'unmatched_quote")
-        except SyntaxError as msg:
-            assert msg.filename == '<string>'
-
-    def test_exec_and_name_lookups(self):
-        ns = {}
-        exec """def f():
-            exec 'x=1' in locals()
-            return x""" in ns
-
-        f = ns['f']
-
-        try:
-            res = f()
-        except NameError as e: # keep py.test from exploding confused
-            raise e
-
-        assert res == 1
-
-    def test_exec_unicode(self):
-        # 's' is a string
-        s = "x = u'\xd0\xb9\xd1\x86\xd1\x83\xd0\xba\xd0\xb5\xd0\xbd'"
-        # 'u' is a unicode
-        u = s.decode('utf-8')
-        exec u
-        assert len(x) == 6
-        assert ord(x[0]) == 0x0439
-        assert ord(x[1]) == 0x0446
-        assert ord(x[2]) == 0x0443
-        assert ord(x[3]) == 0x043a
-        assert ord(x[4]) == 0x0435
-        assert ord(x[5]) == 0x043d
-
-    def test_eval_unicode(self):
-        u = "u'%s'" % unichr(0x1234)
-        v = eval(u)
-        assert v == unichr(0x1234)
-
-    def test_compile_unicode(self):
-        s = "x = u'\xd0\xb9\xd1\x86\xd1\x83\xd0\xba\xd0\xb5\xd0\xbd'"
-        u = s.decode('utf-8')
-        c = compile(u, '<input>', 'exec')
-        exec c
-        assert len(x) == 6
-        assert ord(x[0]) == 0x0439
-
-    def test_nested_qualified_exec(self):
-        import sys
-        if sys.version_info < (2, 7, 9):
-            skip()
-        code = ["""
+def test_nested_qualified_exec():
+    import sys
+    if sys.version_info < (2, 7, 9):
+        skip()
+    code = ["""
 def g():
     def f():
         if True:
             exec "" in {}, {}
-        """, """
+    """, """
 def g():
     def f():
         if True:
             exec("", {}, {})
-        """]
-        for c in code:
-            compile(c, "<code>", "exec")
+    """]
+    for c in code:
+        compile(c, "<code>", "exec")
 
-    def test_exec_tuple(self):
-        # note: this is VERY different than testing exec("a = 42", d), because
-        # this specific case is handled specially by the AST compiler
-        d = {}
-        x = ("a = 42", d)
-        exec x
-        assert d['a'] == 42
+def test_exec_tuple():
+    # note: this is VERY different than testing exec("a = 42", d), because
+    # this specific case is handled specially by the AST compiler
+    d = {}
+    x = ("a = 42", d)
+    exec x
+    assert d['a'] == 42
diff --git a/pypy/interpreter/test/test_exec.py b/pypy/interpreter/test/test_exec.py
--- a/pypy/interpreter/test/test_exec.py
+++ b/pypy/interpreter/test/test_exec.py
@@ -16,257 +16,3 @@
             assert 'abc' in g
             assert 'cba' in g
     ''')
-
-
-class AppTestExecStmt:
-    def test_string(self):
-        g = {}
-        l = {}
-        exec "a = 3" in g, l
-        assert l['a'] == 3
-
-    def test_localfill(self):
-        g = {}
-        exec "a = 3" in g
-        assert g['a'] == 3
-
-    def test_builtinsupply(self):
-        g = {}
-        exec "pass" in g
-        assert g.has_key('__builtins__')
-
-    def test_invalidglobal(self):
-        def f():
-            exec 'pass' in 1
-        raises(TypeError,f)
-
-    def test_invalidlocal(self):
-        def f():
-            exec 'pass' in {}, 2
-        raises(TypeError,f)
-
-    def test_codeobject(self):
-        co = compile("a = 3", '<string>', 'exec')
-        g = {}
-        l = {}
-        exec co in g, l
-        assert l['a'] == 3
-
-    def test_implicit(self):
-        a = 4
-        exec "a = 3"
-        assert a == 3
-
-    def test_tuplelocals(self):
-        g = {}
-        l = {}
-        exec ("a = 3", g, l)
-        assert l['a'] == 3
-
-    def test_tupleglobals(self):
-        g = {}
-        exec ("a = 3", g)
-        assert g['a'] == 3
-
-    def test_exceptionfallthrough(self):
-        def f():
-            exec 'raise TypeError' in {}
-        raises(TypeError,f)
-
-    def test_global_stmt(self):
-        g = {}
-        l = {}
-        co = compile("global a; a=5", '', 'exec')
-        #import dis
-        #dis.dis(co)
-        exec co in g, l
-        assert l == {}
-        assert g['a'] == 5
-
-    def test_specialcase_free_load(self):
-        exec """if 1:
-            def f():
-                exec 'a=3'
-                return a
-            x = f()\n"""
-        assert x == 3
-
-    def test_specialcase_free_load2(self):
-        exec """if 1:
-            def f(a):
-                exec 'a=3'
-                return a
-            x = f(4)\n"""
-        assert x == 3
-
-    def test_specialcase_globals_and_exec(self):
-        d = {}
-        exec """if 1:
-            b = 2
-            c = 3 
-            d = 4 
-            def f(a):
-                global b
-                exec 'd=42 ; b=7'
-                return a,b,c,d
-            #import dis
-            #dis.dis(f)
-            res = f(3)\n""" in d
-        r = d['res']
-        assert r == (3,2,3,42)
-
-    def test_nested_names_are_not_confused(self):
-        def get_nested_class():
-            method_and_var = "var"
-            class Test(object):
-                def method_and_var(self):
-                    return "method"
-                def test(self):
-                    return method_and_var
-                def actual_global(self):
-                    return str("global")
-                def str(self):
-                    return str(self)
-            return Test()
-        t = get_nested_class()
-        assert t.actual_global() == "global" 
-        assert t.test() == 'var'
-        assert t.method_and_var() == 'method'
-
-    def test_import_star_shadows_global(self):
-        d = {'platform' : 3}
-        exec """if 1:
-            def f():
-                from sys import *
-                return platform
-            res = f()\n""" in d
-        import sys
-        assert d['res'] == sys.platform
-
-    def test_import_global_takes_precendence(self):
-        d = {'platform' : 3}
-        exec """if 1:
-            def f():
-                global platform
-                from sys import *
-                return platform
-            res = f()\n""" in d
-        import sys
-        assert d['platform'] == 3
-
-    def test_exec_load_name(self):
-        d = {'x': 2}
-        exec """if 1:
-            def f():
-                save = x 
-                exec "x=3"
-                return x,save
-        \n""" in d
-        res = d['f']()
-        assert res == (3, 2)
-
-    def test_space_bug(self):
-        d = {}
-        exec "x=5 " in d
-        assert d['x'] == 5
-
-    def test_synerr(self):
-        def x():
-            exec "1 2"
-        raises(SyntaxError, x)
-
-    def test_mapping_as_locals(self):
-        import sys
-        if sys.version_info < (2,5) or not hasattr(sys, 'pypy_objspaceclass'):
-            skip("need CPython 2.5 or PyPy for non-dictionaries in exec statements")
-        class M(object):
-            def __getitem__(self, key):
-                return key
-            def __setitem__(self, key, value):
-                self.result[key] = value
-            def setdefault(self, key, value):
-                assert key == '__builtins__'
-        m = M()
-        m.result = {}
-        exec "x=m" in {}, m
-        assert m.result == {'x': 'm'}
-        exec "y=n" in m   # NOTE: this doesn't work in CPython 2.4
-        assert m.result == {'x': 'm', 'y': 'n'}
-
-    def test_filename(self):
-        try:
-            exec "'unmatched_quote"
-        except SyntaxError as msg:
-            assert msg.filename == '<string>'
-        try:
-            eval("'unmatched_quote")
-        except SyntaxError as msg:
-            assert msg.filename == '<string>'
-
-    def test_exec_and_name_lookups(self):
-        ns = {}
-        exec """def f():
-            exec 'x=1' in locals()
-            return x""" in ns
-
-        f = ns['f']
-
-        try:
-            res = f()
-        except NameError as e: # keep py.test from exploding confused
-            raise e
-
-        assert res == 1
-
-    def test_exec_unicode(self):
-        # 's' is a string
-        s = "x = u'\xd0\xb9\xd1\x86\xd1\x83\xd0\xba\xd0\xb5\xd0\xbd'"
-        # 'u' is a unicode
-        u = s.decode('utf-8')
-        exec u
-        assert len(x) == 6
-        assert ord(x[0]) == 0x0439
-        assert ord(x[1]) == 0x0446
-        assert ord(x[2]) == 0x0443
-        assert ord(x[3]) == 0x043a
-        assert ord(x[4]) == 0x0435
-        assert ord(x[5]) == 0x043d
-
-    def test_eval_unicode(self):
-        u = "u'%s'" % unichr(0x1234)
-        v = eval(u)
-        assert v == unichr(0x1234)
-
-    def test_compile_unicode(self):
-        s = "x = u'\xd0\xb9\xd1\x86\xd1\x83\xd0\xba\xd0\xb5\xd0\xbd'"
-        u = s.decode('utf-8')
-        c = compile(u, '<input>', 'exec')
-        exec c
-        assert len(x) == 6
-        assert ord(x[0]) == 0x0439
-
-    def test_nested_qualified_exec(self):
-        import sys
-        if sys.version_info < (2, 7, 9):
-            skip()
-        code = ["""
-def g():
-    def f():
-        if True:
-            exec "" in {}, {}
-        """, """
-def g():
-    def f():
-        if True:
-            exec("", {}, {})
-        """]
-        for c in code:
-            compile(c, "<code>", "exec")
-
-    def test_exec_tuple(self):
-        # note: this is VERY different than testing exec("a = 42", d), because
-        # this specific case is handled specially by the AST compiler
-        d = {}
-        x = ("a = 42", d)
-        exec x
-        assert d['a'] == 42

From pypy.commits at gmail.com  Fri Nov  8 21:53:08 2019
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 08 Nov 2019 18:53:08 -0800 (PST)
Subject: [pypy-commit] pypy default: Test cleanup
Message-ID: <5dc62a14.1c69fb81.98c40.8dcd@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r97998:88d8e29d0fa8
Date: 2019-11-09 02:13 +0000
http://bitbucket.org/pypy/pypy/changeset/88d8e29d0fa8/

Log:	Test cleanup

diff --git a/pypy/interpreter/test/apptest_exec.py b/pypy/interpreter/test/apptest_exec.py
--- a/pypy/interpreter/test/apptest_exec.py
+++ b/pypy/interpreter/test/apptest_exec.py
@@ -2,6 +2,8 @@
 
 New for PyPy - Could be incorporated into CPython regression tests.
 """
+import pytest
+
 def test_string():
     g = {}
     l = {}
@@ -19,14 +21,12 @@
     assert g.has_key('__builtins__')
 
 def test_invalidglobal():
-    def f():
+    with pytest.raises(TypeError):
         exec 'pass' in 1
-    raises(TypeError,f)
 
 def test_invalidlocal():
-    def f():
+    with pytest.raises(TypeError):
         exec 'pass' in {}, 2
-    raises(TypeError,f)
 
 def test_codeobject():
     co = compile("a = 3", '<string>', 'exec')
@@ -52,9 +52,8 @@
     assert g['a'] == 3
 
 def test_exceptionfallthrough():
-    def f():
+    with pytest.raises(TypeError):
         exec 'raise TypeError' in {}
-    raises(TypeError,f)
 
 def test_global_stmt():
     g = {}
@@ -154,14 +153,13 @@
     assert d['x'] == 5
 
 def test_synerr():
-    def x():
+    with pytest.raises(SyntaxError):
         exec "1 2"
-    raises(SyntaxError, x)
 
 def test_mapping_as_locals():
     import sys
-    if sys.version_info < (2,5) or not hasattr(sys, 'pypy_objspaceclass'):
-        skip("need CPython 2.5 or PyPy for non-dictionaries in exec statements")
+    if not hasattr(sys, 'pypy_objspaceclass'):
+        skip("need PyPy for non-dictionaries in exec statements")
     class M(object):
         def __getitem__(self, key):
             return key
@@ -173,18 +171,16 @@
     m.result = {}
     exec "x=m" in {}, m
     assert m.result == {'x': 'm'}
-    exec "y=n" in m   # NOTE: this doesn't work in CPython 2.4
+    exec "y=n" in m   # NOTE: this doesn't work in CPython
     assert m.result == {'x': 'm', 'y': 'n'}
 
 def test_filename():
-    try:
+    with pytest.raises(SyntaxError) as excinfo:
         exec "'unmatched_quote"
-    except SyntaxError as msg:
-        assert msg.filename == '<string>'
-    try:
+    assert excinfo.value.filename == '<string>'
+    with pytest.raises(SyntaxError) as excinfo:
         eval("'unmatched_quote")
-    except SyntaxError as msg:
-        assert msg.filename == '<string>'
+    assert excinfo.value.filename == '<string>'
 
 def test_exec_and_name_lookups():
     ns = {}
@@ -193,13 +189,7 @@
         return x""" in ns
 
     f = ns['f']
-
-    try:
-        res = f()
-    except NameError as e: # keep py.test from exploding confused
-        raise e
-
-    assert res == 1
+    assert f() == 1
 
 def test_exec_unicode():
     # 's' is a string

From pypy.commits at gmail.com  Fri Nov  8 21:53:10 2019
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 08 Nov 2019 18:53:10 -0800 (PST)
Subject: [pypy-commit] pypy py3.6: Convert test_exec to apptest
Message-ID: <5dc62a16.1c69fb81.dde06.6b69@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6
Changeset: r97999:1843e911ad01
Date: 2019-11-09 02:21 +0000
http://bitbucket.org/pypy/pypy/changeset/1843e911ad01/

Log:	Convert test_exec to apptest

diff --git a/pypy/interpreter/test/test_exec.py b/pypy/interpreter/test/apptest_exec.py
rename from pypy/interpreter/test/test_exec.py
rename to pypy/interpreter/test/apptest_exec.py
--- a/pypy/interpreter/test/test_exec.py
+++ b/pypy/interpreter/test/apptest_exec.py
@@ -2,208 +2,203 @@
 
 New for PyPy - Could be incorporated into CPython regression tests.
 """
-from rpython.tool.udir import udir
+def test_string():
+    g = {}
+    l = {}
+    exec("a = 3", g, l)
+    assert l['a'] == 3
 
+def test_localfill():
+    g = {}
+    exec("a = 3", g)
+    assert g['a'] == 3
 
-class AppTestExecStmt: 
+def test_builtinsupply():
+    g = {}
+    exec("pass", g)
+    assert '__builtins__' in g
 
-    def test_string(self):
-        g = {}
-        l = {}
-        exec("a = 3", g, l)
-        assert l['a'] == 3
+def test_invalidglobal():
+    def f():
+        exec('pass', 1)
+    raises(TypeError, f)
 
-    def test_localfill(self):
-        g = {}
-        exec("a = 3", g)
-        assert g['a'] == 3
+def test_invalidlocal():
+    def f():
+        exec('pass', {}, 2)
+    raises(TypeError, f)
 
-    def test_builtinsupply(self):
-        g = {}
-        exec("pass", g)
-        assert '__builtins__' in g
+def test_codeobject():
+    co = compile("a = 3", '<string>', 'exec')
+    g = {}
+    l = {}
+    exec(co, g, l)
+    assert l['a'] == 3
 
-    def test_invalidglobal(self):
+def test_implicit():
+    a = 4
+    exec("a = 3")
+    assert a == 3
+
+def test_tuplelocals():
+    g = {}
+    l = {}
+    exec("a = 3", g, l)
+    assert l['a'] == 3
+
+def test_tupleglobals():
+    g = {}
+    exec("a = 3", g)
+    assert g['a'] == 3
+
+def test_exceptionfallthrough():
+    def f():
+        exec('raise TypeError', {})
+    raises(TypeError, f)
+
+def test_global_stmt():
+    g = {}
+    l = {}
+    co = compile("global a; a=5", '', 'exec')
+    #import dis
+    #dis.dis(co)
+    exec(co, g, l)
+    assert l == {}
+    assert g['a'] == 5
+
+def test_specialcase_free_load():
+    exec("""if 1:
         def f():
-            exec('pass', 1)
-        raises(TypeError, f)
+            exec('a=3')
+            return a
+        raises(NameError, f)\n""")
 
-    def test_invalidlocal(self):
+def test_specialcase_free_load2():
+    exec("""if 1:
+        def f(a):
+            exec('a=3')
+            return a
+        x = f(4)\n""")
+    assert eval("x") == 3
+
+def test_nested_names_are_not_confused():
+    def get_nested_class():
+        method_and_var = "var"
+        class Test(object):
+            def method_and_var(self):
+                return "method"
+            def test(self):
+                return method_and_var
+            def actual_global(self):
+                return str("global")
+            def str(self):
+                return str(self)
+        return Test()
+    t = get_nested_class()
+    assert t.actual_global() == "global"
+    assert t.test() == 'var'
+    assert t.method_and_var() == 'method'
+
+def test_exec_load_name():
+    d = {'x': 2}
+    exec("""if 1:
         def f():
-            exec('pass', {}, 2)
-        raises(TypeError, f)
+            save = x
+            exec("x=3")
+            return x,save
+    \n""", d)
+    res = d['f']()
+    assert res == (2, 2)
 
-    def test_codeobject(self):
-        co = compile("a = 3", '<string>', 'exec')
-        g = {}
-        l = {}
-        exec(co, g, l)
-        assert l['a'] == 3
+def test_space_bug():
+    d = {}
+    exec("x=5 ", d)
+    assert d['x'] == 5
 
-    def test_implicit(self):
-        a = 4
-        exec("a = 3")
-        assert a == 3
+def test_synerr():
+    def x():
+        exec("1 2")
+    raises(SyntaxError, x)
 
-    def test_tuplelocals(self):
-        g = {}
-        l = {}
-        exec("a = 3", g, l)
-        assert l['a'] == 3
+def test_mapping_as_locals():
+    class M(object):
+        def __getitem__(self, key):
+            return key
+        def __setitem__(self, key, value):
+            self.result[key] = value
+        def setdefault(self, key, value):
+            assert key == '__builtins__'
+    m = M()
+    m.result = {}
+    exec("x=m", {}, m)
+    assert m.result == {'x': 'm'}
+    try:
+        exec("y=n", m)
+    except TypeError:
+        pass
+    else:
+        assert False, 'Expected TypeError'
+    raises(TypeError, eval, "m", m)
 
-    def test_tupleglobals(self):
-        g = {}
-        exec("a = 3", g)
-        assert g['a'] == 3
+def test_filename():
+    try:
+        exec("'unmatched_quote")
+    except SyntaxError as msg:
+        assert msg.filename == '<string>'
+    try:
+        eval("'unmatched_quote")
+    except SyntaxError as msg:
+        assert msg.filename == '<string>'
 
-    def test_exceptionfallthrough(self):
-        def f():
-            exec('raise TypeError', {})
-        raises(TypeError, f)
+def test_exec_and_name_lookups():
+    ns = {}
+    exec("""def f():
+        exec('x=1', globals())
+        return x\n""", ns)
 
-    def test_global_stmt(self):
-        g = {}
-        l = {}
-        co = compile("global a; a=5", '', 'exec')
-        #import dis
-        #dis.dis(co)
-        exec(co, g, l)
-        assert l == {}
-        assert g['a'] == 5
+    f = ns['f']
 
-    def test_specialcase_free_load(self):
-        exec("""if 1:
-            def f():
-                exec('a=3')
-                return a
-            raises(NameError, f)\n""")
+    try:
+        res = f()
+    except NameError as e: # keep py.test from exploding confused
+        raise e
 
-    def test_specialcase_free_load2(self):
-        exec("""if 1:
-            def f(a):
-                exec('a=3')
-                return a
-            x = f(4)\n""")
-        assert eval("x") == 3
+    assert res == 1
 
-    def test_nested_names_are_not_confused(self):
-        def get_nested_class():
-            method_and_var = "var"
-            class Test(object):
-                def method_and_var(self):
-                    return "method"
-                def test(self):
-                    return method_and_var
-                def actual_global(self):
-                    return str("global")
-                def str(self):
-                    return str(self)
-            return Test()
-        t = get_nested_class()
-        assert t.actual_global() == "global" 
-        assert t.test() == 'var'
-        assert t.method_and_var() == 'method'
+def test_exec_unicode():
+    # 's' is a bytes string
+    s = b"x = '\xd0\xb9\xd1\x86\xd1\x83\xd0\xba\xd0\xb5\xd0\xbd'"
+    # 'u' is a unicode
+    u = s.decode('utf-8')
+    ns = {}
+    exec(u, ns)
+    x = ns['x']
+    assert len(x) == 6
+    assert ord(x[0]) == 0x0439
+    assert ord(x[1]) == 0x0446
+    assert ord(x[2]) == 0x0443
+    assert ord(x[3]) == 0x043a
+    assert ord(x[4]) == 0x0435
+    assert ord(x[5]) == 0x043d
 
-    def test_exec_load_name(self):
-        d = {'x': 2}
-        exec("""if 1:
-            def f():
-                save = x 
-                exec("x=3")
-                return x,save
-        \n""", d)
-        res = d['f']()
-        assert res == (2, 2)
+def test_eval_unicode():
+    u = "'%s'" % chr(0x1234)
+    v = eval(u)
+    assert v == chr(0x1234)
 
-    def test_space_bug(self):
-        d = {}
-        exec("x=5 ", d)
-        assert d['x'] == 5
+def test_compile_bytes():
+    s = b"x = '\xd0\xb9\xd1\x86\xd1\x83\xd0\xba\xd0\xb5\xd0\xbd'"
+    c = compile(s, '<input>', 'exec')
+    ns = {}
+    exec(c, ns)
+    x = ns['x']
+    assert len(x) == 6
+    assert ord(x[0]) == 0x0439
 
-    def test_synerr(self):
-        def x():
-            exec("1 2")
-        raises(SyntaxError, x)
-
-    def test_mapping_as_locals(self):
-        class M(object):
-            def __getitem__(self, key):
-                return key
-            def __setitem__(self, key, value):
-                self.result[key] = value
-            def setdefault(self, key, value):
-                assert key == '__builtins__'
-        m = M()
-        m.result = {}
-        exec("x=m", {}, m)
-        assert m.result == {'x': 'm'}
-        try:
-            exec("y=n", m)
-        except TypeError:
-            pass
-        else:
-            assert False, 'Expected TypeError'
-        raises(TypeError, eval, "m", m)
-
-    def test_filename(self):
-        try:
-            exec("'unmatched_quote")
-        except SyntaxError as msg:
-            assert msg.filename == '<string>'
-        try:
-            eval("'unmatched_quote")
-        except SyntaxError as msg:
-            assert msg.filename == '<string>'
-
-    def test_exec_and_name_lookups(self):
-        ns = {}
-        exec("""def f():
-            exec('x=1', globals())
-            return x\n""", ns)
-
-        f = ns['f']
-
-        try:
-            res = f()
-        except NameError as e: # keep py.test from exploding confused
-            raise e
-
-        assert res == 1
-
-    def test_exec_unicode(self):
-        # 's' is a bytes string
-        s = b"x = '\xd0\xb9\xd1\x86\xd1\x83\xd0\xba\xd0\xb5\xd0\xbd'"
-        # 'u' is a unicode
-        u = s.decode('utf-8')
-        ns = {}
-        exec(u, ns)
-        x = ns['x']
-        assert len(x) == 6
-        assert ord(x[0]) == 0x0439
-        assert ord(x[1]) == 0x0446
-        assert ord(x[2]) == 0x0443
-        assert ord(x[3]) == 0x043a
-        assert ord(x[4]) == 0x0435
-        assert ord(x[5]) == 0x043d
-
-    def test_eval_unicode(self):
-        u = "'%s'" % chr(0x1234)
-        v = eval(u)
-        assert v == chr(0x1234)
-
-    def test_compile_bytes(self):
-        s = b"x = '\xd0\xb9\xd1\x86\xd1\x83\xd0\xba\xd0\xb5\xd0\xbd'"
-        c = compile(s, '<input>', 'exec')
-        ns = {}
-        exec(c, ns)
-        x = ns['x']
-        assert len(x) == 6
-        assert ord(x[0]) == 0x0439
-
-    def test_issue3297(self):
-        c = compile("a, b = '\U0001010F', '\\U0001010F'", "dummy", "exec")
-        d = {}
-        exec(c, d)
-        assert d['a'] == d['b']
-        assert len(d['a']) == len(d['b'])
-        assert ascii(d['a']) == ascii(d['b'])
+def test_issue3297():
+    c = compile("a, b = '\U0001010F', '\\U0001010F'", "dummy", "exec")
+    d = {}
+    exec(c, d)
+    assert d['a'] == d['b']
+    assert len(d['a']) == len(d['b'])
+    assert ascii(d['a']) == ascii(d['b'])

From pypy.commits at gmail.com  Fri Nov  8 21:53:11 2019
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 08 Nov 2019 18:53:11 -0800 (PST)
Subject: [pypy-commit] pypy py3.6: Test cleanup
Message-ID: <5dc62a17.1c69fb81.ee299.a534@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6
Changeset: r98000:dbbea25a4b3e
Date: 2019-11-09 02:38 +0000
http://bitbucket.org/pypy/pypy/changeset/dbbea25a4b3e/

Log:	Test cleanup

diff --git a/pypy/interpreter/test/apptest_exec.py b/pypy/interpreter/test/apptest_exec.py
--- a/pypy/interpreter/test/apptest_exec.py
+++ b/pypy/interpreter/test/apptest_exec.py
@@ -2,6 +2,8 @@
 
 New for PyPy - Could be incorporated into CPython regression tests.
 """
+import pytest
+
 def test_string():
     g = {}
     l = {}
@@ -19,14 +21,12 @@
     assert '__builtins__' in g
 
 def test_invalidglobal():
-    def f():
+    with pytest.raises(TypeError):
         exec('pass', 1)
-    raises(TypeError, f)
 
 def test_invalidlocal():
-    def f():
+    with pytest.raises(TypeError):
         exec('pass', {}, 2)
-    raises(TypeError, f)
 
 def test_codeobject():
     co = compile("a = 3", '<string>', 'exec')
@@ -52,9 +52,8 @@
     assert g['a'] == 3
 
 def test_exceptionfallthrough():
-    def f():
+    with pytest.raises(TypeError):
         exec('raise TypeError', {})
-    raises(TypeError, f)
 
 def test_global_stmt():
     g = {}
@@ -67,11 +66,12 @@
     assert g['a'] == 5
 
 def test_specialcase_free_load():
-    exec("""if 1:
-        def f():
-            exec('a=3')
-            return a
-        raises(NameError, f)\n""")
+    def f():
+        exec('a=3')
+        return a
+
+    with raises(NameError):
+        f()
 
 def test_specialcase_free_load2():
     exec("""if 1:
@@ -116,9 +116,8 @@
     assert d['x'] == 5
 
 def test_synerr():
-    def x():
+    with pytest.raises(SyntaxError):
         exec("1 2")
-    raises(SyntaxError, x)
 
 def test_mapping_as_locals():
     class M(object):
@@ -132,23 +131,18 @@
     m.result = {}
     exec("x=m", {}, m)
     assert m.result == {'x': 'm'}
-    try:
+    with raises(TypeError):
         exec("y=n", m)
-    except TypeError:
-        pass
-    else:
-        assert False, 'Expected TypeError'
-    raises(TypeError, eval, "m", m)
+    with raises(TypeError):
+        eval("m", m)
 
 def test_filename():
-    try:
+    with pytest.raises(SyntaxError) as excinfo:
         exec("'unmatched_quote")
-    except SyntaxError as msg:
-        assert msg.filename == '<string>'
-    try:
+    assert excinfo.value.filename == '<string>'
+    with pytest.raises(SyntaxError) as excinfo:
         eval("'unmatched_quote")
-    except SyntaxError as msg:
-        assert msg.filename == '<string>'
+    assert excinfo.value.filename == '<string>'
 
 def test_exec_and_name_lookups():
     ns = {}
@@ -157,13 +151,7 @@
         return x\n""", ns)
 
     f = ns['f']
-
-    try:
-        res = f()
-    except NameError as e: # keep py.test from exploding confused
-        raise e
-
-    assert res == 1
+    assert f() == 1
 
 def test_exec_unicode():
     # 's' is a bytes string

From pypy.commits at gmail.com  Fri Nov  8 22:06:03 2019
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 08 Nov 2019 19:06:03 -0800 (PST)
Subject: [pypy-commit] pypy py3.6: fix tests to pass on CPython
Message-ID: <5dc62d1b.1c69fb81.a7195.c807@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6
Changeset: r98002:cfec65b692b9
Date: 2019-11-09 03:05 +0000
http://bitbucket.org/pypy/pypy/changeset/cfec65b692b9/

Log:	fix tests to pass on CPython

diff --git a/pypy/interpreter/test/apptest_exec.py b/pypy/interpreter/test/apptest_exec.py
--- a/pypy/interpreter/test/apptest_exec.py
+++ b/pypy/interpreter/test/apptest_exec.py
@@ -38,7 +38,7 @@
 def test_implicit():
     a = 4
     exec("a = 3")
-    assert a == 3
+    assert a == 4
 
 def test_tuplelocals():
     g = {}
@@ -79,7 +79,7 @@
             exec('a=3')
             return a
         x = f(4)\n""")
-    assert eval("x") == 3
+    assert eval("x") == 4
 
 def test_nested_names_are_not_confused():
     def get_nested_class():

From pypy.commits at gmail.com  Sat Nov  9 05:54:00 2019
From: pypy.commits at gmail.com (arigo)
Date: Sat, 09 Nov 2019 02:54:00 -0800 (PST)
Subject: [pypy-commit] pypy sandbox-2: Add "unicodedata"
Message-ID: <5dc69ac8.1c69fb81.786bb.89a8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: sandbox-2
Changeset: r98004:1dafe2678e0d
Date: 2019-11-09 11:52 +0100
http://bitbucket.org/pypy/pypy/changeset/1dafe2678e0d/

Log:	Add "unicodedata"

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -43,7 +43,7 @@
 # --sandbox
 sandbox_modules = default_modules.copy()
 sandbox_modules.update([
-    "struct", "cStringIO", "itertools", "array", "binascii",
+    "struct", "cStringIO", "itertools", "array", "binascii", "unicodedata",
 ])
 
 import rpython.rlib.rvmprof.cintf

From pypy.commits at gmail.com  Sat Nov  9 05:54:02 2019
From: pypy.commits at gmail.com (arigo)
Date: Sat, 09 Nov 2019 02:54:02 -0800 (PST)
Subject: [pypy-commit] pypy py3.6-sandbox-2: hg merge sandbox-2
Message-ID: <5dc69aca.1c69fb81.8bc5c.2598@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.6-sandbox-2
Changeset: r98005:9cfae941846c
Date: 2019-11-09 11:52 +0100
http://bitbucket.org/pypy/pypy/changeset/9cfae941846c/

Log:	hg merge sandbox-2

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -47,7 +47,7 @@
 # --sandbox
 sandbox_modules = default_modules.copy()
 sandbox_modules.update([
-    "array", "binascii",
+    "array", "binascii", "unicodedata",
 ])
 
 import rpython.rlib.rvmprof.cintf

From pypy.commits at gmail.com  Sat Nov  9 12:02:28 2019
From: pypy.commits at gmail.com (stevie_92)
Date: Sat, 09 Nov 2019 09:02:28 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-gc-cycle: Implemented working set for
 linked objects during rrc marking
Message-ID: <5dc6f124.1c69fb81.9a05e.0c8c@mx.google.com>

Author: Stefan Beyer <home at sbeyer.at>
Branch: cpyext-gc-cycle
Changeset: r98006:a3a571806246
Date: 2019-11-09 18:01 +0100
http://bitbucket.org/pypy/pypy/changeset/a3a571806246/

Log:	Implemented working set for linked objects during rrc marking
	Removed debug output

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -2136,6 +2136,12 @@
             #
             hdr.tid |= GCFLAG_VISITED
             #
+            if self.rrc_enabled: # TODO: is this safe here?
+                if self.rrc_gc.state == RawRefCountBaseGC.STATE_MARKING:
+                    self.rrc_gc.visit_pyobj(obj)
+                elif self.rrc_gc.state == RawRefCountBaseGC.STATE_GARBAGE_MARKING:
+                    hdr.tid |= GCFLAG_GARBAGE
+            #
             self.surviving_pinned_objects.append(
                 llarena.getfakearenaaddress(obj - size_gc_header))
             self.pinned_objects_in_nursery += 1
@@ -2403,11 +2409,6 @@
                 self.collect_nonstack_roots()
                 self.visit_all_objects()
                 #
-                # If enabled, do a major collection step for rrc objects.
-                # TODO: move up before "if remaining >= estimate // 2" to
-                #       improve pause times, issues:
-                #         - (non-inc) mark expects all objects to be marked
-                #         - both do not rescan nonstack-roots
                 if self.rrc_enabled:
                     debug_print("starting rrc state:", self.rrc_gc.state)
                     debug_print("starting marking_state:", self.rrc_gc.marking_state)
@@ -2742,9 +2743,11 @@
         # to also set TRACK_YOUNG_PTRS here, for the write barrier.
         hdr.tid |= GCFLAG_VISITED | GCFLAG_TRACK_YOUNG_PTRS
 
-        if self.rrc_enabled and \
-                self.rrc_gc.state == RawRefCountBaseGC.STATE_GARBAGE_MARKING:
-            hdr.tid |= GCFLAG_GARBAGE
+        if self.rrc_enabled:
+            if self.rrc_gc.state == RawRefCountBaseGC.STATE_MARKING:
+                self.rrc_gc.visit_pyobj(obj)
+            elif self.rrc_gc.state == RawRefCountBaseGC.STATE_GARBAGE_MARKING:
+                hdr.tid |= GCFLAG_GARBAGE
 
         if self.has_gcptr(llop.extract_ushort(llgroup.HALFWORD, hdr.tid)):
             #
diff --git a/rpython/memory/gc/rrc/base.py b/rpython/memory/gc/rrc/base.py
--- a/rpython/memory/gc/rrc/base.py
+++ b/rpython/memory/gc/rrc/base.py
@@ -143,7 +143,7 @@
             if not self.gc.is_young_object(obj):
                 lst = self.p_list_old
                 if self.state == self.STATE_MARKING:
-                    debug_print("added p_list", pyobject)
+                    #debug_print("added p_list", pyobject)
                     self.p_list_old_added.append(pyobject)
         lst.append(pyobject)
         dct.setitem(obj, pyobject)
@@ -307,7 +307,7 @@
         if surviving:
             surviving_list.append(pyobject)
             if working_set:
-                debug_print("added p_list", pyobject)
+                #debug_print("added p_list", pyobject)
                 self.p_list_old_added.append(pyobject)
         else:
             self._free(pyobject)
diff --git a/rpython/memory/gc/rrc/incmark.py b/rpython/memory/gc/rrc/incmark.py
--- a/rpython/memory/gc/rrc/incmark.py
+++ b/rpython/memory/gc/rrc/incmark.py
@@ -25,6 +25,7 @@
 
             self.state = self.STATE_MARKING
             self.marking_state = 0
+            self.pyobj_to_trace = self.gc.AddressStack()
             return False
 
         if self.state == self.STATE_MARKING:
@@ -38,7 +39,6 @@
                 return False
             elif self.marking_state == 1:
                 # initialize working set from roots, then pause
-                self.pyobj_to_trace = self.gc.AddressStack()
                 for i in range(0, self.total_objs):
                     obj = self.snapshot_objs[i]
                     self._mark_rawrefcount_obj(obj)
@@ -104,6 +104,27 @@
         debug_print("time mark p_list_old", time.time() - start)
         return True
 
+    def visit_pyobj(self, gcobj):
+        # if there is a pyobj, add it to the working set
+        if self.gc.is_in_nursery(gcobj):
+            dct = self.p_dict_nurs # is this even possible?
+        else:
+            dct = self.p_dict
+        pyobject = dct.get(gcobj)
+        if pyobject <> llmemory.NULL:
+            pyobj = self._pyobj(pyobject)
+            gchdr = self.pyobj_as_gc(pyobj)
+            if gchdr <> lltype.nullptr(self.PYOBJ_GC_HDR):
+                if gchdr.c_gc_refs != self.RAWREFCOUNT_REFS_REACHABLE and \
+                        gchdr.c_gc_refs != self.RAWREFCOUNT_REFS_UNTRACKED:  # object is in snapshot
+                    pass
+                    c_gc_refs = self._pyobj_gc_refcnt_get(gchdr)
+                    index = c_gc_refs - 1
+                    snapobj = self.snapshot_objs[index]
+                    if snapobj.refcnt == 0:
+                        addr = llmemory.cast_ptr_to_adr(snapobj)
+                        self.pyobj_to_trace.append(addr)
+
     def _sync_snapshot(self):
         # sync snapshot with pyob_list:
         #  * check the consistency of "dead" objects and keep all of them
@@ -140,9 +161,6 @@
                 consistent = pyobj.c_ob_refcnt == snapobj.refcnt_original
                 if not consistent:
                     break
-                # move to separate list
-                #self.p_list_old.remove(snapobj.pyobj) # TODO: this might be evil... do something different... -> unlink? special link?
-
                 # remove link, to free non-gc (so they won't get marked and are freed)
                 pyobj = llmemory.cast_adr_to_ptr(snapobj.pyobj, self.PYOBJ_HDR_PTR)
                 link = llmemory.cast_int_to_adr(pyobj.c_ob_pypy_link)
@@ -175,7 +193,6 @@
             else:
                 # new object, keep alive
                 self._pyobj_gc_refcnt_set(pygchdr, 1)
-                #pygchdr.c_gc_refs = 1 << self.RAWREFCOUNT_REFS_SHIFT
                 pyobj = self.gc_as_pyobj(pygchdr)
                 if pyobj.c_ob_pypy_link != 0:
                     addr = llmemory.cast_int_to_adr(pyobj.c_ob_pypy_link)
@@ -219,13 +236,11 @@
             pygchdr = pygchdr_continue_gc
             while pygchdr <> self.pyobj_list:
                 self._pyobj_gc_refcnt_set(pygchdr, 1)
-                #pygchdr.c_gc_refs = 1 << self.RAWREFCOUNT_REFS_SHIFT
                 pygchdr = pygchdr.c_gc_next
             pygchdr = self.pyobj_old_list.c_gc_next
             # resurrect "dead" objects
             while pygchdr <> self.pyobj_old_list:
                 self._pyobj_gc_refcnt_set(pygchdr, 1)
-                #pygchdr.c_gc_refs = 1 << self.RAWREFCOUNT_REFS_SHIFT
                 pygchdr = pygchdr.c_gc_next
             # merge lists
             if not self._gc_list_is_empty(self.pyobj_old_list):
@@ -249,12 +264,10 @@
             pygchdr = pygchdr_continue_isolate
             while pygchdr <> self.pyobj_isolate_old_list:
                 self._pyobj_gc_refcnt_set(pygchdr, 1)
-                #pygchdr.c_gc_refs = 1 << self.RAWREFCOUNT_REFS_SHIFT
                 pygchdr = pygchdr.c_gc_next
             # resurrect "dead" objects
             while pygchdr <> self.pyobj_isolate_dead_list:
                 self._pyobj_gc_refcnt_set(pygchdr, 1)
-                #pygchdr.c_gc_refs = 1 << self.RAWREFCOUNT_REFS_SHIFT
                 pygchdr = pygchdr.c_gc_next
             # merge lists
             if not self._gc_list_is_empty(self.pyobj_isolate_old_list):
@@ -269,9 +282,7 @@
     def _check_consistency_gc(self, pygchdr, pylist_dead_target):
         c_gc_refs = self._pyobj_gc_refcnt_get(pygchdr)
         snapobj = self.snapshot_objs[c_gc_refs - 1]
-        #snapobj = self.snapshot_objs[pygchdr.c_gc_refs - 1]
         self._pyobj_gc_refcnt_set(pygchdr, snapobj.refcnt)
-        #pygchdr.c_gc_refs = snapobj.refcnt
         if snapobj.refcnt == 0:  # object considered dead
             # check consistency (dead subgraphs can never change):
             pyobj = self.gc_as_pyobj(pygchdr)
@@ -358,9 +369,7 @@
                 simple_limit += 1
                 if simple_limit > self.inc_limit: # TODO: add test
                     reached_limit = True
-            self.gc.visit_all_objects()  # TODO: implement sane limit (ex. half of normal limit), retrace proxies
-            self.p_list_old.foreach(self._mark_rawrefcount_linked, None)
-            self.o_list_old.foreach(self._mark_rawrefcount_linked, None)
+            self.gc.visit_all_objects()  # TODO: implement sane limit
             first = False
         return not reached_limit # are there any objects left?
 
diff --git a/rpython/memory/gc/rrc/mark.py b/rpython/memory/gc/rrc/mark.py
--- a/rpython/memory/gc/rrc/mark.py
+++ b/rpython/memory/gc/rrc/mark.py
@@ -12,6 +12,7 @@
 
         if self.state == self.STATE_DEFAULT:
             self.state = self.STATE_MARKING
+            self.pyobj_to_trace = self.gc.AddressStack()
 
         # First, untrack all tuples with only non-gc rrc objects and promote
         # all other tuples to the pyobj_list
@@ -67,6 +68,21 @@
         self.state = self.STATE_DEFAULT
         return True
 
+    def visit_pyobj(self, gcobj):
+        # if there is a pyobj, add it to the working set
+        if self.gc.is_in_nursery(gcobj):
+            dct = self.p_dict_nurs # is this even possible?
+        else:
+            dct = self.p_dict
+        pyobject = dct.get(gcobj)
+        if pyobject <> llmemory.NULL:
+            pyobj = self._pyobj(pyobject)
+            gchdr = self.pyobj_as_gc(pyobj)
+            if gchdr <> lltype.nullptr(self.PYOBJ_GC_HDR):
+                if gchdr.c_gc_refs >> self.RAWREFCOUNT_REFS_SHIFT == 0:
+                    addr = llmemory.cast_ptr_to_adr(gchdr)
+                    self.pyobj_to_trace.append(addr)
+
     def to_obj(self, pyobject):
         if self.use_refcntdict:
             obj = self.pypy_link_dict.get(pyobject)
@@ -165,7 +181,6 @@
         pyobj_old = self.pyobj_list
 
         # initialize working set
-        self.pyobj_to_trace = self.gc.AddressStack()
         gchdr = self.pyobj_old_list.c_gc_next
         while gchdr <> self.pyobj_old_list:
             next_old = gchdr.c_gc_next
@@ -189,8 +204,6 @@
                 gchdr.c_gc_refs += 1 << self.RAWREFCOUNT_REFS_SHIFT
                 self._mark_rawrefcount_obj(gchdr, pyobj_old)
             self.gc.visit_all_objects()
-            self.p_list_old.foreach(self._mark_rawrefcount_linked, None)
-            self.o_list_old.foreach(self._mark_rawrefcount_linked, None)
 
         # now all rawrefcounted objects, which are alive, have a cyclic
         # refcount > 0 or are marked
@@ -230,4 +243,4 @@
             if gchdr <> lltype.nullptr(self.PYOBJ_GC_HDR):
                 if gchdr.c_gc_refs >> self.RAWREFCOUNT_REFS_SHIFT == 0:
                     addr = llmemory.cast_ptr_to_adr(gchdr)
-                    self.pyobj_to_trace.append(addr)
+                    self.pyobj_to_trace.append(addr)
\ No newline at end of file

From pypy.commits at gmail.com  Sat Nov  9 18:23:13 2019
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 09 Nov 2019 15:23:13 -0800 (PST)
Subject: [pypy-commit] pypy default: Prevent all -A tests from being skipped
Message-ID: <5dc74a61.1c69fb81.786bb.04e1@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r98007:8330eedbb70c
Date: 2019-11-09 23:22 +0000
http://bitbucket.org/pypy/pypy/changeset/8330eedbb70c/

Log:	Prevent all -A tests from being skipped

diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -185,7 +185,9 @@
             from pypy.tool.pytest.objspace import gettestobjspace
             # Make cls.space and cls.runappdirect available in tests.
             spaceconfig = getattr(appclass.obj, 'spaceconfig', {})
-            spaceconfig.setdefault('objspace.std.reinterpretasserts', True)
+            config = item.config
+            if not (config.getoption('runappdirect') or config.getoption('direct_apptest')):
+                spaceconfig.setdefault('objspace.std.reinterpretasserts', True)
             appclass.obj.space = gettestobjspace(**spaceconfig)
             appclass.obj.runappdirect = option.runappdirect
 

From pypy.commits at gmail.com  Sun Nov 10 02:39:47 2019
From: pypy.commits at gmail.com (arigo)
Date: Sat, 09 Nov 2019 23:39:47 -0800 (PST)
Subject: [pypy-commit] pypy default: Fix the error message: point to the
 right "cd" directory
Message-ID: <5dc7bec3.1c69fb81.6897f.8386@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r98008:789b5384030f
Date: 2019-11-10 08:38 +0100
http://bitbucket.org/pypy/pypy/changeset/789b5384030f/

Log:	Fix the error message: point to the right "cd" directory

diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -10,7 +10,7 @@
     import os
     msg = "\n\nThe _ssl cffi module either doesn't exist or is incompatible with your machine's shared libraries.\n" + \
           "If you have a compiler installed, you can try to rebuild it by running:\n" + \
-          "cd %s\n" % os.path.abspath(os.path.dirname(os.path.dirname(__file__))) + \
+          "cd %s\n" % os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) + \
           "%s _ssl_build.py\n" % sys.executable
     raise ImportError(str(e) + msg)
 

From pypy.commits at gmail.com  Sun Nov 10 02:42:23 2019
From: pypy.commits at gmail.com (arigo)
Date: Sat, 09 Nov 2019 23:42:23 -0800 (PST)
Subject: [pypy-commit] pypy default: update to cffi/eebc6733b38d
Message-ID: <5dc7bf5f.1c69fb81.93438.fc8a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r98009:6b181a284b2c
Date: 2019-11-10 08:41 +0100
http://bitbucket.org/pypy/pypy/changeset/6b181a284b2c/

Log:	update to cffi/eebc6733b38d

diff --git a/extra_tests/cffi_tests/cffi0/test_verify.py b/extra_tests/cffi_tests/cffi0/test_verify.py
--- a/extra_tests/cffi_tests/cffi0/test_verify.py
+++ b/extra_tests/cffi_tests/cffi0/test_verify.py
@@ -4,6 +4,7 @@
 import sys, os, math, weakref
 from cffi import FFI, VerificationError, VerificationMissing, model, FFIError
 from extra_tests.cffi_tests.support import *
+from extra_tests.cffi_tests.support import extra_compile_args
 
 
 lib_m = ['m']
@@ -14,17 +15,6 @@
         lib_m = ['msvcrt']
     pass      # no obvious -Werror equivalent on MSVC
 else:
-    if (sys.platform == 'darwin' and
-          [int(x) for x in os.uname()[2].split('.')] >= [11, 0, 0]):
-        # assume a standard clang or gcc
-        extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion']
-        # special things for clang
-        extra_compile_args.append('-Qunused-arguments')
-    else:
-        # assume a standard gcc
-        extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion',
-                              '-Wno-unused-parameter']
-
     class FFI(FFI):
         def verify(self, *args, **kwds):
             return super(FFI, self).verify(
diff --git a/extra_tests/cffi_tests/cffi1/test_recompiler.py b/extra_tests/cffi_tests/cffi1/test_recompiler.py
--- a/extra_tests/cffi_tests/cffi1/test_recompiler.py
+++ b/extra_tests/cffi_tests/cffi1/test_recompiler.py
@@ -35,8 +35,9 @@
         source = 'extern "C" {\n%s\n}' % (source,)
     elif sys.platform != 'win32':
         # add '-Werror' to the existing 'extra_compile_args' flags
+        from extra_tests.cffi_tests.support import extra_compile_args
         kwds['extra_compile_args'] = (kwds.get('extra_compile_args', []) +
-                                      ['-Werror'])
+                                      extra_compile_args)
     return _verify(ffi, module_name, source, *args, **kwds)
 
 def test_set_source_no_slashes():
@@ -2039,7 +2040,7 @@
     ffi.cdef("float _Complex f1(float a, float b);");
     lib = verify(ffi, "test_function_returns_float_complex", """
         #include <complex.h>
-        static float _Complex f1(float a, float b) { return a + I*2.0*b; }
+        static float _Complex f1(float a, float b) { return a + I*2.0f*b; }
     """, no_cpp=True)    # <complex.h> fails on some systems with C++
     result = lib.f1(1.25, 5.1)
     assert type(result) == complex
diff --git a/extra_tests/cffi_tests/cffi1/test_verify1.py b/extra_tests/cffi_tests/cffi1/test_verify1.py
--- a/extra_tests/cffi_tests/cffi1/test_verify1.py
+++ b/extra_tests/cffi_tests/cffi1/test_verify1.py
@@ -5,7 +5,7 @@
 from cffi import CDefError
 from cffi import recompiler
 from extra_tests.cffi_tests.support import *
-from extra_tests.cffi_tests.support import _verify
+from extra_tests.cffi_tests.support import _verify, extra_compile_args
 import _cffi_backend
 
 lib_m = ['m']
@@ -14,18 +14,6 @@
     import distutils.ccompiler
     if distutils.ccompiler.get_default_compiler() == 'msvc':
         lib_m = ['msvcrt']
-    extra_compile_args = []      # no obvious -Werror equivalent on MSVC
-else:
-    if (sys.platform == 'darwin' and
-          [int(x) for x in os.uname()[2].split('.')] >= [11, 0, 0]):
-        # assume a standard clang or gcc
-        extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion']
-        # special things for clang
-        extra_compile_args.append('-Qunused-arguments')
-    else:
-        # assume a standard gcc
-        extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion',
-                              '-Wno-unused-parameter']
 
 class FFI(FFI):
     error = _cffi_backend.FFI.error
diff --git a/extra_tests/cffi_tests/support.py b/extra_tests/cffi_tests/support.py
--- a/extra_tests/cffi_tests/support.py
+++ b/extra_tests/cffi_tests/support.py
@@ -1,5 +1,5 @@
 # Generated by pypy/tool/import_cffi.py
-import sys
+import sys, os
 
 if sys.version_info < (3,):
     __all__ = ['u']
@@ -87,3 +87,24 @@
         if not name.startswith('_') and not hasattr(module.ffi, name):
             setattr(ffi, name, NotImplemented)
     return module.lib
+
+
+# For testing, we call gcc with "-Werror".  This is fragile because newer
+# versions of gcc are always better at producing warnings, particularly for
+# auto-generated code.  We need here to adapt and silence them as needed.
+
+if sys.platform == 'win32':
+    extra_compile_args = []      # no obvious -Werror equivalent on MSVC
+else:
+    if (sys.platform == 'darwin' and
+          [int(x) for x in os.uname()[2].split('.')] >= [11, 0, 0]):
+        # assume a standard clang or gcc
+        extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion',
+                              '-Wno-unreachable-code']
+        # special things for clang
+        extra_compile_args.append('-Qunused-arguments')
+    else:
+        # assume a standard gcc
+        extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion',
+                              '-Wno-unused-parameter',
+                              '-Wno-unreachable-code']
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.13.1
+Version: 1.13.2
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -5,8 +5,8 @@
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 from .error import PkgConfigError
 
-__version__ = "1.13.1"
-__version_info__ = (1, 13, 1)
+__version__ = "1.13.2"
+__version_info__ = (1, 13, 2)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -261,12 +261,12 @@
         return (int)_cffi_to_c_wchar3216_t(o);
 }
 
-_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char32_t(int x)
+_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char32_t(unsigned int x)
 {
     if (sizeof(_cffi_wchar_t) == 4)
         return _cffi_from_c_wchar_t((_cffi_wchar_t)x);
     else
-        return _cffi_from_c_wchar3216_t(x);
+        return _cffi_from_c_wchar3216_t((int)x);
 }
 
 
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -224,7 +224,7 @@
 
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.13.1"
+                               "\ncompiled with cffi version: 1.13.2"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -159,9 +159,9 @@
 def _warn_for_non_extern_non_static_global_variable(decl):
     if not decl.storage:
         import warnings
-        warnings.warn("Declaration of global variable '%s' in cdef() should "
-                      "be marked 'extern' for consistency (or possibly "
-                      "'static' in API mode)" % (decl.name,))
+        warnings.warn("Global variable '%s' in cdef(): for consistency "
+                      "with C it should have a storage class specifier "
+                      "(usually 'extern')" % (decl.name,))
 
 def _preprocess(csource):
     # Remove comments.  NOTE: this only work because the cdef() section
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -1,1 +1,1 @@
-VERSION = "1.13.1"
+VERSION = "1.13.2"
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.13.1", ("This test_c.py file is for testing a version"
+assert __version__ == "1.13.2", ("This test_c.py file is for testing a version"
                                  " of cffi that differs from the one that we"
                                  " get from 'import _cffi_backend'")
 if sys.version_info < (3,):

From pypy.commits at gmail.com  Sun Nov 10 18:16:33 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 10 Nov 2019 15:16:33 -0800 (PST)
Subject: [pypy-commit] pypy default: MAXCODE is 65535 when CODESIZE is 2
 (win32)
Message-ID: <5dc89a51.1c69fb81.8f91b.6e0e@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r98010:08b530929d7b
Date: 2019-11-10 16:43 -0500
http://bitbucket.org/pypy/pypy/changeset/08b530929d7b/

Log:	MAXCODE is 65535 when CODESIZE is 2 (win32)

diff --git a/rpython/rlib/rsre/rsre_core.py b/rpython/rlib/rsre/rsre_core.py
--- a/rpython/rlib/rsre/rsre_core.py
+++ b/rpython/rlib/rsre/rsre_core.py
@@ -96,8 +96,10 @@
     def __init__(self, pattern):
         self.pattern = pattern
         # check we don't get the old value of MAXREPEAT
-        # during the untranslated tests
-        if not we_are_translated():
+        # during the untranslated tests. 
+        # On python3, MAXCODE can appear in patterns. It will be 65535
+        # when CODESIZE is 2
+        if not we_are_translated() and rsre_char.CODESIZE != 2:
             assert 65535 not in pattern
 
     def pat(self, index):

From pypy.commits at gmail.com  Sun Nov 10 18:16:35 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 10 Nov 2019 15:16:35 -0800 (PST)
Subject: [pypy-commit] pypy default: use strigified _MSC_VER for the
 compiler version, like CPython
Message-ID: <5dc89a53.1c69fb81.98460.199e@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r98011:f18d4c3047ec
Date: 2019-11-09 01:43 +0200
http://bitbucket.org/pypy/pypy/changeset/f18d4c3047ec/

Log:	use strigified _MSC_VER for the compiler version, like CPython

diff --git a/rpython/rlib/compilerinfo.py b/rpython/rlib/compilerinfo.py
--- a/rpython/rlib/compilerinfo.py
+++ b/rpython/rlib/compilerinfo.py
@@ -19,7 +19,7 @@
 
 if platform.name == 'msvc':
     # XXX hard-code the MSC version, I don't feel like computing it dynamically
-    _C_COMPILER_INFO = '"MSC v.%d 32 bit"' % (platform.version * 10 + 600)
+    _C_COMPILER_INFO = '"MSC v." Py_STR(_MSC_VER)'
 else:
     _C_COMPILER_INFO = '("GCC " __VERSION__)'
 
diff --git a/rpython/translator/c/src/commondefs.h b/rpython/translator/c/src/commondefs.h
--- a/rpython/translator/c/src/commondefs.h
+++ b/rpython/translator/c/src/commondefs.h
@@ -123,3 +123,7 @@
 #  define MS_WINDOWS    /* a synonym */
 #endif
 #endif
+
+/* stringify a constant */
+#define Py_XSTR(x)   #x
+#define Py_STR(x)  Py_XSTR(x)

From pypy.commits at gmail.com  Sun Nov 10 18:16:37 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 10 Nov 2019 15:16:37 -0800 (PST)
Subject: [pypy-commit] pypy msvc-discovery: simplify,
 use only setuptools.msvc to find msvc compilers
Message-ID: <5dc89a55.1c69fb81.e6994.2813@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: msvc-discovery
Changeset: r98012:f53664167f74
Date: 2019-11-08 21:55 +0200
http://bitbucket.org/pypy/pypy/changeset/f53664167f74/

Log:	simplify, use only setuptools.msvc to find msvc compilers

diff --git a/rpython/tool/setuptools_msvc.py b/rpython/tool/setuptools_msvc.py
--- a/rpython/tool/setuptools_msvc.py
+++ b/rpython/tool/setuptools_msvc.py
@@ -11,35 +11,39 @@
 Microsoft Visual C++ 10.0:
     Microsoft Windows SDK 7.1 (x86, x64, ia64)
 
-Microsoft Visual C++ 14.0:
+Microsoft Visual C++ 14.X:
     Microsoft Visual C++ Build Tools 2015 (x86, x64, arm)
-    Microsoft Visual Studio 2017 (x86, x64, arm, arm64)
     Microsoft Visual Studio Build Tools 2017 (x86, x64, arm, arm64)
+    Microsoft Visual Studio Build Tools 2019 (x86, x64, arm, arm64)
+
+This may also support compilers shipped with compatible Visual Studio versions.
 """
 
-# copied as-is from python3
-# modified to:
+# copied as-is from setuptools v41.6.0 setuptools/msvc.py
+# modified to (search for `"modified"):
 #    - commented out monkey patching
 #    - lookup() returns str not unicode
 
-import os
+import json
+from io import open
+from os import listdir, pathsep
+from os.path import join, isfile, isdir, dirname
 import sys
 import platform
 import itertools
 import distutils.errors
+from setuptools.extern.packaging.version import LegacyVersion
 
 from setuptools.extern.six.moves import filterfalse
 
-#from .monkey import get_unpatched
+# modified
+# from .monkey import get_unpatched
 
 if platform.system() == 'Windows':
     from setuptools.extern.six.moves import winreg
-    safe_env = os.environ
+    from os import environ
 else:
-    """
-    Mock winreg and environ so the module can be imported
-    on this platform.
-    """
+    # Mock winreg and environ so the module can be imported on this platform.
 
     class winreg:
         HKEY_USERS = None
@@ -47,12 +51,12 @@
         HKEY_LOCAL_MACHINE = None
         HKEY_CLASSES_ROOT = None
 
-    safe_env = dict()
+    environ = dict()
 
 _msvc9_suppress_errors = (
     # msvc9compiler isn't available on some platforms
     ImportError,
-    
+
     # msvc9compiler raises DistutilsPlatformError in some
     # environments. See #1118.
     distutils.errors.DistutilsPlatformError,
@@ -67,16 +71,14 @@
 def msvc9_find_vcvarsall(version):
     """
     Patched "distutils.msvc9compiler.find_vcvarsall" to use the standalone
-    compiler build for Python (VCForPython). Fall back to original behavior
-    when the standalone compiler is not available.
+    compiler build for Python
+    (VCForPython / Microsoft Visual C++ Compiler for Python 2.7).
+
+    Fall back to original behavior when the standalone compiler is not
+    available.
 
     Redirect the path of "vcvarsall.bat".
 
-    Known supported compilers
-    -------------------------
-    Microsoft Visual C++ 9.0:
-        Microsoft Visual C++ Compiler for Python 2.7 (x86, amd64)
-
     Parameters
     ----------
     version: float
@@ -84,24 +86,25 @@
 
     Return
     ------
-    vcvarsall.bat path: str
+    str
+        vcvarsall.bat path
     """
-    VC_BASE = r'Software\%sMicrosoft\DevDiv\VCForPython\%0.1f'
-    key = VC_BASE % ('', version)
+    vc_base = r'Software\%sMicrosoft\DevDiv\VCForPython\%0.1f'
+    key = vc_base % ('', version)
     try:
         # Per-user installs register the compiler path here
         productdir = Reg.get_value(key, "installdir")
     except KeyError:
         try:
             # All-user installs on a 64-bit system register here
-            key = VC_BASE % ('Wow6432Node\\', version)
+            key = vc_base % ('Wow6432Node\\', version)
             productdir = Reg.get_value(key, "installdir")
         except KeyError:
             productdir = None
 
     if productdir:
-        vcvarsall = os.path.os.path.join(productdir, "vcvarsall.bat")
-        if os.path.isfile(vcvarsall):
+        vcvarsall = join(productdir, "vcvarsall.bat")
+        if isfile(vcvarsall):
             return vcvarsall
 
     return get_unpatched(msvc9_find_vcvarsall)(version)
@@ -110,20 +113,10 @@
 def msvc9_query_vcvarsall(ver, arch='x86', *args, **kwargs):
     """
     Patched "distutils.msvc9compiler.query_vcvarsall" for support extra
-    compilers.
+    Microsoft Visual C++ 9.0 and 10.0 compilers.
 
     Set environment without use of "vcvarsall.bat".
 
-    Known supported compilers
-    -------------------------
-    Microsoft Visual C++ 9.0:
-        Microsoft Visual C++ Compiler for Python 2.7 (x86, amd64)
-        Microsoft Windows SDK 6.1 (x86, x64, ia64)
-        Microsoft Windows SDK 7.0 (x86, x64, ia64)
-
-    Microsoft Visual C++ 10.0:
-        Microsoft Windows SDK 7.1 (x86, x64, ia64)
-
     Parameters
     ----------
     ver: float
@@ -133,9 +126,10 @@
 
     Return
     ------
-    environment: dict
+    dict
+        environment
     """
-    # Try to get environement from vcvarsall.bat (Classical way)
+    # Try to get environment from vcvarsall.bat (Classical way)
     #try:
     #    orig = get_unpatched(msvc9_query_vcvarsall)
     #    return orig(ver, arch, *args, **kwargs)
@@ -157,17 +151,10 @@
 def msvc14_get_vc_env(plat_spec):
     """
     Patched "distutils._msvccompiler._get_vc_env" for support extra
-    compilers.
+    Microsoft Visual C++ 14.X compilers.
 
     Set environment without use of "vcvarsall.bat".
 
-    Known supported compilers
-    -------------------------
-    Microsoft Visual C++ 14.0:
-        Microsoft Visual C++ Build Tools 2015 (x86, x64, arm)
-        Microsoft Visual Studio 2017 (x86, x64, arm, arm64)
-        Microsoft Visual Studio Build Tools 2017 (x86, x64, arm, arm64)
-
     Parameters
     ----------
     plat_spec: str
@@ -175,7 +162,8 @@
 
     Return
     ------
-    environment: dict
+    dict
+        environment
     """
     # Try to get environment from vcvarsall.bat (Classical way)
     #try:
@@ -200,7 +188,6 @@
     """
     if "numpy.distutils" in sys.modules:
         import numpy as np
-        from pkg_resources.extern.packaging.version import LegacyVersion
         if LegacyVersion(np.__version__) < LegacyVersion('1.11.2'):
             return np.distutils.ccompiler.gen_lib_options(*args, **kwargs)
     return get_unpatched(msvc14_gen_lib_options)(*args, **kwargs)
@@ -222,9 +209,9 @@
         if version == 9.0:
             if arch.lower().find('ia64') > -1:
                 # For VC++ 9.0, if IA64 support is needed, redirect user
-                # to Windows SDK 7.0
-                message += ' Get it with "Microsoft Windows SDK 7.0": '
-                message += msdownload % 3138
+                # to Windows SDK 7.0.
+                # Note: No download link available from Microsoft.
+                message += ' Get it with "Microsoft Windows SDK 7.0"'
             else:
                 # For VC++ 9.0 redirect user to Vc++ for Python 2.7 :
                 # This redirection link is maintained by Microsoft.
@@ -235,36 +222,59 @@
             message += ' Get it with "Microsoft Windows SDK 7.1": '
             message += msdownload % 8279
         elif version >= 14.0:
-            # For VC++ 14.0 Redirect user to Visual C++ Build Tools
-            message += (' Get it with "Microsoft Visual C++ Build Tools": '
-                        r'http://landinghub.visualstudio.com/'
-                        'visual-cpp-build-tools')
+            # For VC++ 14.X Redirect user to latest Visual C++ Build Tools
+            message += (' Get it with "Build Tools for Visual Studio": '
+                        r'https://visualstudio.microsoft.com/downloads/')
 
     exc.args = (message, )
 
 
 class PlatformInfo:
     """
-    Current and Target Architectures informations.
+    Current and Target Architectures information.
 
     Parameters
     ----------
     arch: str
         Target architecture.
     """
-    current_cpu = safe_env.get('processor_architecture', '').lower()
+    current_cpu = environ.get('processor_architecture', '').lower()
 
     def __init__(self, arch):
         self.arch = arch.lower().replace('x64', 'amd64')
 
     @property
     def target_cpu(self):
+        """
+        Return Target CPU architecture.
+
+        Return
+        ------
+        str
+            Target CPU
+        """
         return self.arch[self.arch.find('_') + 1:]
 
     def target_is_x86(self):
+        """
+        Return True if target CPU is x86 32 bits..
+
+        Return
+        ------
+        bool
+            CPU is x86 32 bits
+        """
         return self.target_cpu == 'x86'
 
     def current_is_x86(self):
+        """
+        Return True if current CPU is x86 32 bits..
+
+        Return
+        ------
+        bool
+            CPU is x86 32 bits
+        """
         return self.current_cpu == 'x86'
 
     def current_dir(self, hidex86=False, x64=False):
@@ -280,8 +290,8 @@
 
         Return
         ------
-        subfolder: str
-            '\target', or '' (see hidex86 parameter)
+        str
+            subfolder: '\target', or '' (see hidex86 parameter)
         """
         return (
             '' if (self.current_cpu == 'x86' and hidex86) else
@@ -302,8 +312,8 @@
 
         Return
         ------
-        subfolder: str
-            '\current', or '' (see hidex86 parameter)
+        str
+            subfolder: '\current', or '' (see hidex86 parameter)
         """
         return (
             '' if (self.target_cpu == 'x86' and hidex86) else
@@ -318,13 +328,13 @@
         Parameters
         ----------
         forcex86: bool
-            Use 'x86' as current architecture even if current acritecture is
+            Use 'x86' as current architecture even if current architecture is
             not x86.
 
         Return
         ------
-        subfolder: str
-            '' if target architecture is current architecture,
+        str
+            subfolder: '' if target architecture is current architecture,
             '\current_target' if not.
         """
         current = 'x86' if forcex86 else self.current_cpu
@@ -336,7 +346,7 @@
 
 class RegistryInfo:
     """
-    Microsoft Visual Studio related registry informations.
+    Microsoft Visual Studio related registry information.
 
     Parameters
     ----------
@@ -355,6 +365,11 @@
     def visualstudio(self):
         """
         Microsoft Visual Studio root registry key.
+
+        Return
+        ------
+        str
+            Registry key
         """
         return 'VisualStudio'
 
@@ -362,27 +377,47 @@
     def sxs(self):
         """
         Microsoft Visual Studio SxS registry key.
+
+        Return
+        ------
+        str
+            Registry key
         """
-        return os.path.join(self.visualstudio, 'SxS')
+        return join(self.visualstudio, 'SxS')
 
     @property
     def vc(self):
         """
         Microsoft Visual C++ VC7 registry key.
+
+        Return
+        ------
+        str
+            Registry key
         """
-        return os.path.join(self.sxs, 'VC7')
+        return join(self.sxs, 'VC7')
 
     @property
     def vs(self):
         """
         Microsoft Visual Studio VS7 registry key.
+
+        Return
+        ------
+        str
+            Registry key
         """
-        return os.path.join(self.sxs, 'VS7')
+        return join(self.sxs, 'VS7')
 
     @property
     def vc_for_python(self):
         """
         Microsoft Visual C++ for Python registry key.
+
+        Return
+        ------
+        str
+            Registry key
         """
         return r'DevDiv\VCForPython'
 
@@ -390,6 +425,11 @@
     def microsoft_sdk(self):
         """
         Microsoft SDK registry key.
+
+        Return
+        ------
+        str
+            Registry key
         """
         return 'Microsoft SDKs'
 
@@ -397,20 +437,35 @@
     def windows_sdk(self):
         """
         Microsoft Windows/Platform SDK registry key.
+
+        Return
+        ------
+        str
+            Registry key
         """
-        return os.path.join(self.microsoft_sdk, 'Windows')
+        return join(self.microsoft_sdk, 'Windows')
 
     @property
     def netfx_sdk(self):
         """
         Microsoft .NET Framework SDK registry key.
+
+        Return
+        ------
+        str
+            Registry key
         """
-        return os.path.join(self.microsoft_sdk, 'NETFXSDK')
+        return join(self.microsoft_sdk, 'NETFXSDK')
 
     @property
     def windows_kits_roots(self):
         """
         Microsoft Windows Kits Roots registry key.
+
+        Return
+        ------
+        str
+            Registry key
         """
         return r'Windows Kits\Installed Roots'
 
@@ -427,10 +482,11 @@
 
         Return
         ------
-        str: value
+        str
+            Registry key
         """
         node64 = '' if self.pi.current_is_x86() or x86 else 'Wow6432Node'
-        return os.path.join('Software', node64, 'Microsoft', key)
+        return join('Software', node64, 'Microsoft', key)
 
     def lookup(self, key, name):
         """
@@ -445,18 +501,19 @@
 
         Return
         ------
-        str: value
+        str
+            value
         """
-        KEY_READ = winreg.KEY_READ
+        key_read = winreg.KEY_READ
         openkey = winreg.OpenKey
         ms = self.microsoft
         for hkey in self.HKEYS:
             try:
-                bkey = openkey(hkey, ms(key), 0, KEY_READ)
+                bkey = openkey(hkey, ms(key), 0, key_read)
             except (OSError, IOError):
                 if not self.pi.current_is_x86():
                     try:
-                        bkey = openkey(hkey, ms(key, True), 0, KEY_READ)
+                        bkey = openkey(hkey, ms(key, True), 0, key_read)
                     except (OSError, IOError):
                         continue
                 else:
@@ -470,7 +527,7 @@
 
 class SystemInfo:
     """
-    Microsoft Windows and Visual Studio related system inormations.
+    Microsoft Windows and Visual Studio related system information.
 
     Parameters
     ----------
@@ -481,30 +538,52 @@
     """
 
     # Variables and properties in this class use originals CamelCase variables
-    # names from Microsoft source files for more easy comparaison.
-    WinDir = safe_env.get('WinDir', '')
-    ProgramFiles = safe_env.get('ProgramFiles', '')
-    ProgramFilesx86 = safe_env.get('ProgramFiles(x86)', ProgramFiles)
+    # names from Microsoft source files for more easy comparison.
+    WinDir = environ.get('WinDir', '')
+    ProgramFiles = environ.get('ProgramFiles', '')
+    ProgramFilesx86 = environ.get('ProgramFiles(x86)', ProgramFiles)
 
     def __init__(self, registry_info, vc_ver=None):
         self.ri = registry_info
         self.pi = self.ri.pi
-        self.vc_ver = vc_ver or self._find_latest_available_vc_ver()
 
-    def _find_latest_available_vc_ver(self):
-        try:
-            return self.find_available_vc_vers()[-1]
-        except IndexError:
-            err = 'No Microsoft Visual C++ version found'
-            raise distutils.errors.DistutilsPlatformError(err)
+        self.known_vs_paths = self.find_programdata_vs_vers()
 
-    def find_available_vc_vers(self):
+        # Except for VS15+, VC version is aligned with VS version
+        self.vs_ver = self.vc_ver = (
+                vc_ver or self._find_latest_available_vs_ver())
+
+    def _find_latest_available_vs_ver(self):
         """
-        Find all available Microsoft Visual C++ versions.
+        Find the latest VC version
+
+        Return
+        ------
+        float
+            version
+        """
+        reg_vc_vers = self.find_reg_vs_vers()
+
+        if not (reg_vc_vers or self.known_vs_paths):
+            raise distutils.errors.DistutilsPlatformError(
+                'No Microsoft Visual C++ version found')
+
+        vc_vers = set(reg_vc_vers)
+        vc_vers.update(self.known_vs_paths)
+        return sorted(vc_vers)[-1]
+
+    def find_reg_vs_vers(self):
+        """
+        Find Microsoft Visual Studio versions available in registry.
+
+        Return
+        ------
+        list of float
+            Versions
         """
         ms = self.ri.microsoft
         vckeys = (self.ri.vc, self.ri.vc_for_python, self.ri.vs)
-        vc_vers = []
+        vs_vers = []
         for hkey in self.ri.HKEYS:
             for key in vckeys:
                 try:
@@ -515,49 +594,109 @@
                 for i in range(values):
                     try:
                         ver = float(winreg.EnumValue(bkey, i)[0])
-                        if ver not in vc_vers:
-                            vc_vers.append(ver)
+                        if ver not in vs_vers:
+                            vs_vers.append(ver)
                     except ValueError:
                         pass
                 for i in range(subkeys):
                     try:
                         ver = float(winreg.EnumKey(bkey, i))
-                        if ver not in vc_vers:
-                            vc_vers.append(ver)
+                        if ver not in vs_vers:
+                            vs_vers.append(ver)
                     except ValueError:
                         pass
-        return sorted(vc_vers)
+        return sorted(vs_vers)
+
+    def find_programdata_vs_vers(self):
+        r"""
+        Find Visual studio 2017+ versions from information in
+        "C:\ProgramData\Microsoft\VisualStudio\Packages\_Instances".
+
+        Return
+        ------
+        dict
+            float version as key, path as value.
+        """
+        vs_versions = {}
+        instances_dir = \
+            r'C:\ProgramData\Microsoft\VisualStudio\Packages\_Instances'
+
+        try:
+            hashed_names = listdir(instances_dir)
+
+        except (OSError, IOError):
+            # Directory not exists with all Visual Studio versions
+            return vs_versions
+
+        for name in hashed_names:
+            try:
+                # Get VS installation path from "state.json" file
+                state_path = join(instances_dir, name, 'state.json')
+                with open(state_path, 'rt', encoding='utf-8') as state_file:
+                    state = json.load(state_file)
+                vs_path = state['installationPath']
+
+                # Raises OSError if this VS installation does not contain VC
+                listdir(join(vs_path, r'VC\Tools\MSVC'))
+
+                # Store version and path
+                # modified for python2 to encode paths
+                vs_versions[self._as_float_version(
+                    state['installationVersion'])] = vs_path.encode('utf-8')
+
+            except (OSError, IOError, KeyError):
+                # Skip if "state.json" file is missing or bad format
+                continue
+
+        return vs_versions
+
+    @staticmethod
+    def _as_float_version(version):
+        """
+        Return a string version as a simplified float version (major.minor)
+
+        Parameters
+        ----------
+        version: str
+            Version.
+
+        Return
+        ------
+        float
+            version
+        """
+        return float('.'.join(version.split('.')[:2]))
 
     @property
     def VSInstallDir(self):
         """
         Microsoft Visual Studio directory.
+
+        Return
+        ------
+        str
+            path
         """
         # Default path
-        name = 'Microsoft Visual Studio %0.1f' % self.vc_ver
-        default = os.path.join(self.ProgramFilesx86, name)
+        default = join(self.ProgramFilesx86,
+                       'Microsoft Visual Studio %0.1f' % self.vs_ver)
 
         # Try to get path from registry, if fail use default path
-        return self.ri.lookup(self.ri.vs, '%0.1f' % self.vc_ver) or default
+        return self.ri.lookup(self.ri.vs, '%0.1f' % self.vs_ver) or default
 
     @property
     def VCInstallDir(self):
         """
         Microsoft Visual C++ directory.
+
+        Return
+        ------
+        str
+            path
         """
-        self.VSInstallDir
+        path = self._guess_vc() or self._guess_vc_legacy()
 
-        guess_vc = self._guess_vc() or self._guess_vc_legacy()
-
-        # Try to get "VC++ for Python" path from registry as default path
-        reg_path = os.path.join(self.ri.vc_for_python, '%0.1f' % self.vc_ver)
-        python_vc = self.ri.lookup(reg_path, 'installdir')
-        default_vc = os.path.join(python_vc, 'VC') if python_vc else guess_vc
-
-        # Try to get path from registry, if fail use default path
-        path = self.ri.lookup(self.ri.vc, '%0.1f' % self.vc_ver) or default_vc
-
-        if not os.path.isdir(path):
+        if not isdir(path):
             msg = 'Microsoft Visual C++ directory not found'
             raise distutils.errors.DistutilsPlatformError(msg)
 
@@ -565,186 +704,256 @@
 
     def _guess_vc(self):
         """
-        Locate Visual C for 2017
+        Locate Visual C++ for VS2017+.
+
+        Return
+        ------
+        str
+            path
         """
-        if self.vc_ver <= 14.0:
-            return
+        if self.vs_ver <= 14.0:
+            return ''
 
-        default = r'VC\Tools\MSVC'
-        guess_vc = os.path.join(self.VSInstallDir, default)
+        try:
+            # First search in known VS paths
+            vs_dir = self.known_vs_paths[self.vs_ver]
+        except KeyError:
+            # Else, search with path from registry
+            vs_dir = self.VSInstallDir
+
+        guess_vc = join(vs_dir, r'VC\Tools\MSVC')
+
         # Subdir with VC exact version as name
         try:
-            vc_exact_ver = os.listdir(guess_vc)[-1]
-            return os.path.join(guess_vc, vc_exact_ver)
+            # Update the VC version with real one instead of VS version
+            vc_ver = listdir(guess_vc)[-1]
+            self.vc_ver = self._as_float_version(vc_ver)
+            return join(guess_vc, vc_ver)
         except (OSError, IOError, IndexError):
-            pass
+            return ''
 
     def _guess_vc_legacy(self):
         """
-        Locate Visual C for versions prior to 2017
+        Locate Visual C++ for versions prior to 2017.
+
+        Return
+        ------
+        str
+            path
         """
-        default = r'Microsoft Visual Studio %0.1f\VC' % self.vc_ver
-        return os.path.join(self.ProgramFilesx86, default)
+        default = join(self.ProgramFilesx86,
+                       r'Microsoft Visual Studio %0.1f\VC' % self.vs_ver)
+
+        # Try to get "VC++ for Python" path from registry as default path
+        reg_path = join(self.ri.vc_for_python, '%0.1f' % self.vs_ver)
+        python_vc = self.ri.lookup(reg_path, 'installdir')
+        default_vc = join(python_vc, 'VC') if python_vc else default
+
+        # Try to get path from registry, if fail use default path
+        return self.ri.lookup(self.ri.vc, '%0.1f' % self.vs_ver) or default_vc
 
     @property
     def WindowsSdkVersion(self):
         """
         Microsoft Windows SDK versions for specified MSVC++ version.
+
+        Return
+        ------
+        tuple of str
+            versions
         """
-        if self.vc_ver <= 9.0:
-            return ('7.0', '6.1', '6.0a')
-        elif self.vc_ver == 10.0:
-            return ('7.1', '7.0a')
-        elif self.vc_ver == 11.0:
-            return ('8.0', '8.0a')
-        elif self.vc_ver == 12.0:
-            return ('8.1', '8.1a')
-        elif self.vc_ver >= 14.0:
-            return ('10.0', '8.1')
+        if self.vs_ver <= 9.0:
+            return '7.0', '6.1', '6.0a'
+        elif self.vs_ver == 10.0:
+            return '7.1', '7.0a'
+        elif self.vs_ver == 11.0:
+            return '8.0', '8.0a'
+        elif self.vs_ver == 12.0:
+            return '8.1', '8.1a'
+        elif self.vs_ver >= 14.0:
+            return '10.0', '8.1'
 
     @property
     def WindowsSdkLastVersion(self):
         """
-        Microsoft Windows SDK last version
+        Microsoft Windows SDK last version.
+
+        Return
+        ------
+        str
+            version
         """
-        return self._use_last_dir_name(os.path.join(
-            self.WindowsSdkDir, 'lib'))
+        return self._use_last_dir_name(join(self.WindowsSdkDir, 'lib'))
 
     @property
     def WindowsSdkDir(self):
         """
         Microsoft Windows SDK directory.
+
+        Return
+        ------
+        str
+            path
         """
         sdkdir = ''
         for ver in self.WindowsSdkVersion:
             # Try to get it from registry
-            loc = os.path.join(self.ri.windows_sdk, 'v%s' % ver)
+            loc = join(self.ri.windows_sdk, 'v%s' % ver)
             sdkdir = self.ri.lookup(loc, 'installationfolder')
             if sdkdir:
                 break
-        if not sdkdir or not os.path.isdir(sdkdir):
+        if not sdkdir or not isdir(sdkdir):
             # Try to get "VC++ for Python" version from registry
-            path = os.path.join(self.ri.vc_for_python, '%0.1f' % self.vc_ver)
+            path = join(self.ri.vc_for_python, '%0.1f' % self.vc_ver)
             install_base = self.ri.lookup(path, 'installdir')
             if install_base:
-                sdkdir = os.path.join(install_base, 'WinSDK')
-        if not sdkdir or not os.path.isdir(sdkdir):
+                sdkdir = join(install_base, 'WinSDK')
+        if not sdkdir or not isdir(sdkdir):
             # If fail, use default new path
             for ver in self.WindowsSdkVersion:
                 intver = ver[:ver.rfind('.')]
-                path = r'Microsoft SDKs\Windows Kits\%s' % (intver)
-                d = os.path.join(self.ProgramFiles, path)
-                if os.path.isdir(d):
+                path = r'Microsoft SDKs\Windows Kits\%s' % intver
+                d = join(self.ProgramFiles, path)
+                if isdir(d):
                     sdkdir = d
-        if not sdkdir or not os.path.isdir(sdkdir):
+        if not sdkdir or not isdir(sdkdir):
             # If fail, use default old path
             for ver in self.WindowsSdkVersion:
                 path = r'Microsoft SDKs\Windows\v%s' % ver
-                d = os.path.join(self.ProgramFiles, path)
-                if os.path.isdir(d):
+                d = join(self.ProgramFiles, path)
+                if isdir(d):
                     sdkdir = d
         if not sdkdir:
             # If fail, use Platform SDK
-            sdkdir = os.path.join(self.VCInstallDir, 'PlatformSDK')
+            sdkdir = join(self.VCInstallDir, 'PlatformSDK')
         return sdkdir
 
     @property
     def WindowsSDKExecutablePath(self):
         """
         Microsoft Windows SDK executable directory.
+
+        Return
+        ------
+        str
+            path
         """
         # Find WinSDK NetFx Tools registry dir name
-        if self.vc_ver <= 11.0:
+        if self.vs_ver <= 11.0:
             netfxver = 35
             arch = ''
         else:
             netfxver = 40
-            hidex86 = True if self.vc_ver <= 12.0 else False
+            hidex86 = True if self.vs_ver <= 12.0 else False
             arch = self.pi.current_dir(x64=True, hidex86=hidex86)
         fx = 'WinSDK-NetFx%dTools%s' % (netfxver, arch.replace('\\', '-'))
 
-        # liste all possibles registry paths
+        # list all possibles registry paths
         regpaths = []
-        if self.vc_ver >= 14.0:
+        if self.vs_ver >= 14.0:
             for ver in self.NetFxSdkVersion:
-                regpaths += [os.path.join(self.ri.netfx_sdk, ver, fx)]
+                regpaths += [join(self.ri.netfx_sdk, ver, fx)]
 
         for ver in self.WindowsSdkVersion:
-            regpaths += [os.path.join(self.ri.windows_sdk, 'v%sA' % ver, fx)]
+            regpaths += [join(self.ri.windows_sdk, 'v%sA' % ver, fx)]
 
         # Return installation folder from the more recent path
         for path in regpaths:
             execpath = self.ri.lookup(path, 'installationfolder')
             if execpath:
-                break
-        return execpath
+                return execpath
 
     @property
     def FSharpInstallDir(self):
         """
         Microsoft Visual F# directory.
+
+        Return
+        ------
+        str
+            path
         """
-        path = r'%0.1f\Setup\F#' % self.vc_ver
-        path = os.path.join(self.ri.visualstudio, path)
+        path = join(self.ri.visualstudio, r'%0.1f\Setup\F#' % self.vs_ver)
         return self.ri.lookup(path, 'productdir') or ''
 
     @property
     def UniversalCRTSdkDir(self):
         """
         Microsoft Universal CRT SDK directory.
+
+        Return
+        ------
+        str
+            path
         """
         # Set Kit Roots versions for specified MSVC++ version
-        if self.vc_ver >= 14.0:
-            vers = ('10', '81')
-        else:
-            vers = ()
+        vers = ('10', '81') if self.vs_ver >= 14.0 else ()
 
         # Find path of the more recent Kit
         for ver in vers:
             sdkdir = self.ri.lookup(self.ri.windows_kits_roots,
                                     'kitsroot%s' % ver)
             if sdkdir:
-                break
-        return sdkdir or ''
+                return sdkdir or ''
 
     @property
     def UniversalCRTSdkLastVersion(self):
         """
-        Microsoft Universal C Runtime SDK last version
+        Microsoft Universal C Runtime SDK last version.
+
+        Return
+        ------
+        str
+            version
         """
-        return self._use_last_dir_name(os.path.join(
-            self.UniversalCRTSdkDir, 'lib'))
+        return self._use_last_dir_name(join(self.UniversalCRTSdkDir, 'lib'))
 
     @property
     def NetFxSdkVersion(self):
         """
         Microsoft .NET Framework SDK versions.
+
+        Return
+        ------
+        tuple of str
+            versions
         """
-        # Set FxSdk versions for specified MSVC++ version
-        if self.vc_ver >= 14.0:
-            return ('4.6.1', '4.6')
-        else:
-            return ()
+        # Set FxSdk versions for specified VS version
+        return (('4.7.2', '4.7.1', '4.7',
+                 '4.6.2', '4.6.1', '4.6',
+                 '4.5.2', '4.5.1', '4.5')
+                if self.vs_ver >= 14.0 else ())
 
     @property
     def NetFxSdkDir(self):
         """
         Microsoft .NET Framework SDK directory.
+
+        Return
+        ------
+        str
+            path
         """
+        sdkdir = ''
         for ver in self.NetFxSdkVersion:
-            loc = os.path.join(self.ri.netfx_sdk, ver)
+            loc = join(self.ri.netfx_sdk, ver)
             sdkdir = self.ri.lookup(loc, 'kitsinstallationfolder')
             if sdkdir:
                 break
-        return sdkdir or ''
+        return sdkdir
 
     @property
     def FrameworkDir32(self):
         """
         Microsoft .NET Framework 32bit directory.
+
+        Return
+        ------
+        str
+            path
         """
         # Default path
-        guess_fw = os.path.join(self.WinDir, r'Microsoft.NET\Framework')
+        guess_fw = join(self.WinDir, r'Microsoft.NET\Framework')
 
         # Try to get path from registry, if fail use default path
         return self.ri.lookup(self.ri.vc, 'frameworkdir32') or guess_fw
@@ -753,9 +962,14 @@
     def FrameworkDir64(self):
         """
         Microsoft .NET Framework 64bit directory.
+
+        Return
+        ------
+        str
+            path
         """
         # Default path
-        guess_fw = os.path.join(self.WinDir, r'Microsoft.NET\Framework64')
+        guess_fw = join(self.WinDir, r'Microsoft.NET\Framework64')
 
         # Try to get path from registry, if fail use default path
         return self.ri.lookup(self.ri.vc, 'frameworkdir64') or guess_fw
@@ -764,6 +978,11 @@
     def FrameworkVersion32(self):
         """
         Microsoft .NET Framework 32bit versions.
+
+        Return
+        ------
+        tuple of str
+            versions
         """
         return self._find_dot_net_versions(32)
 
@@ -771,6 +990,11 @@
     def FrameworkVersion64(self):
         """
         Microsoft .NET Framework 64bit versions.
+
+        Return
+        ------
+        tuple of str
+            versions
         """
         return self._find_dot_net_versions(64)
 
@@ -782,6 +1006,11 @@
         ----------
         bits: int
             Platform number of bits: 32 or 64.
+
+        Return
+        ------
+        tuple of str
+            versions
         """
         # Find actual .NET version in registry
         reg_ver = self.ri.lookup(self.ri.vc, 'frameworkver%d' % bits)
@@ -789,18 +1018,17 @@
         ver = reg_ver or self._use_last_dir_name(dot_net_dir, 'v') or ''
 
         # Set .NET versions for specified MSVC++ version
-        if self.vc_ver >= 12.0:
-            frameworkver = (ver, 'v4.0')
-        elif self.vc_ver >= 10.0:
-            frameworkver = ('v4.0.30319' if ver.lower()[:2] != 'v4' else ver,
-                            'v3.5')
-        elif self.vc_ver == 9.0:
-            frameworkver = ('v3.5', 'v2.0.50727')
-        if self.vc_ver == 8.0:
-            frameworkver = ('v3.0', 'v2.0.50727')
-        return frameworkver
+        if self.vs_ver >= 12.0:
+            return ver, 'v4.0'
+        elif self.vs_ver >= 10.0:
+            return 'v4.0.30319' if ver.lower()[:2] != 'v4' else ver, 'v3.5'
+        elif self.vs_ver == 9.0:
+            return 'v3.5', 'v2.0.50727'
+        elif self.vs_ver == 8.0:
+            return 'v3.0', 'v2.0.50727'
 
-    def _use_last_dir_name(self, path, prefix=''):
+    @staticmethod
+    def _use_last_dir_name(path, prefix=''):
         """
         Return name of the last dir in path or '' if no dir found.
 
@@ -809,12 +1037,17 @@
         path: str
             Use dirs in this path
         prefix: str
-            Use only dirs startings by this prefix
+            Use only dirs starting by this prefix
+
+        Return
+        ------
+        str
+            name
         """
         matching_dirs = (
             dir_name
-            for dir_name in reversed(os.listdir(path))
-            if os.path.isdir(os.path.join(path, dir_name)) and
+            for dir_name in reversed(listdir(path))
+            if isdir(join(path, dir_name)) and
             dir_name.startswith(prefix)
         )
         return next(matching_dirs, None) or ''
@@ -825,7 +1058,7 @@
     Return environment variables for specified Microsoft Visual C++ version
     and platform : Lib, Include, Path and libpath.
 
-    This function is compatible with Microsoft Visual C++ 9.0 to 14.0.
+    This function is compatible with Microsoft Visual C++ 9.0 to 14.X.
 
     Script created by analysing Microsoft environment configuration files like
     "vcvars[...].bat", "SetEnv.Cmd", "vcbuildtools.bat", ...
@@ -842,7 +1075,7 @@
     """
 
     # Variables and properties in this class use originals CamelCase variables
-    # names from Microsoft source files for more easy comparaison.
+    # names from Microsoft source files for more easy comparison.
 
     def __init__(self, arch, vc_ver=None, vc_min_ver=0):
         self.pi = PlatformInfo(arch)
@@ -854,204 +1087,254 @@
             raise distutils.errors.DistutilsPlatformError(err)
 
     @property
+    def vs_ver(self):
+        """
+        Microsoft Visual Studio.
+
+        Return
+        ------
+        float
+            version
+        """
+        return self.si.vs_ver
+
+    @property
     def vc_ver(self):
         """
         Microsoft Visual C++ version.
+
+        Return
+        ------
+        float
+            version
         """
         return self.si.vc_ver
 
     @property
     def VSTools(self):
         """
-        Microsoft Visual Studio Tools
+        Microsoft Visual Studio Tools.
+
+        Return
+        ------
+        list of str
+            paths
         """
         paths = [r'Common7\IDE', r'Common7\Tools']
 
-        if self.vc_ver >= 14.0:
+        if self.vs_ver >= 14.0:
             arch_subdir = self.pi.current_dir(hidex86=True, x64=True)
             paths += [r'Common7\IDE\CommonExtensions\Microsoft\TestWindow']
             paths += [r'Team Tools\Performance Tools']
             paths += [r'Team Tools\Performance Tools%s' % arch_subdir]
 
-        return [os.path.join(self.si.VSInstallDir, path) for path in paths]
+        return [join(self.si.VSInstallDir, path) for path in paths]
 
     @property
     def VCIncludes(self):
         """
-        Microsoft Visual C++ & Microsoft Foundation Class Includes
+        Microsoft Visual C++ & Microsoft Foundation Class Includes.
+
+        Return
+        ------
+        list of str
+            paths
         """
-        return [os.path.join(self.si.VCInstallDir, 'Include'),
-                os.path.join(self.si.VCInstallDir, r'ATLMFC\Include')]
+        return [join(self.si.VCInstallDir, 'Include'),
+                join(self.si.VCInstallDir, r'ATLMFC\Include')]
 
     @property
     def VCLibraries(self):
         """
-        Microsoft Visual C++ & Microsoft Foundation Class Libraries
+        Microsoft Visual C++ & Microsoft Foundation Class Libraries.
+
+        Return
+        ------
+        list of str
+            paths
         """
-        if self.vc_ver >= 15.0:
+        if self.vs_ver >= 15.0:
             arch_subdir = self.pi.target_dir(x64=True)
         else:
             arch_subdir = self.pi.target_dir(hidex86=True)
         paths = ['Lib%s' % arch_subdir, r'ATLMFC\Lib%s' % arch_subdir]
 
-        if self.vc_ver >= 14.0:
+        if self.vs_ver >= 14.0:
             paths += [r'Lib\store%s' % arch_subdir]
 
-        return [os.path.join(self.si.VCInstallDir, path) for path in paths]
+        return [join(self.si.VCInstallDir, path) for path in paths]
 
     @property
     def VCStoreRefs(self):
         """
-        Microsoft Visual C++ store references Libraries
+        Microsoft Visual C++ store references Libraries.
+
+        Return
+        ------
+        list of str
+            paths
         """
-        if self.vc_ver < 14.0:
+        if self.vs_ver < 14.0:
             return []
-        return [os.path.join(self.si.VCInstallDir, r'Lib\store\references')]
+        return [join(self.si.VCInstallDir, r'Lib\store\references')]
 
     @property
     def VCTools(self):
         """
-        Microsoft Visual C++ Tools
+        Microsoft Visual C++ Tools.
+
+        Return
+        ------
+        list of str
+            paths
         """
         si = self.si
-        tools = [os.path.join(si.VCInstallDir, 'VCPackages')]
+        tools = [join(si.VCInstallDir, 'VCPackages')]
 
-        forcex86 = True if self.vc_ver <= 10.0 else False
+        forcex86 = True if self.vs_ver <= 10.0 else False
         arch_subdir = self.pi.cross_dir(forcex86)
         if arch_subdir:
-            tools += [os.path.join(si.VCInstallDir, 'Bin%s' % arch_subdir)]
+            tools += [join(si.VCInstallDir, 'Bin%s' % arch_subdir)]
 
-        if self.vc_ver == 14.0:
+        if self.vs_ver == 14.0:
             path = 'Bin%s' % self.pi.current_dir(hidex86=True)
-            tools += [os.path.join(si.VCInstallDir, path)]
+            tools += [join(si.VCInstallDir, path)]
 
-        elif self.vc_ver >= 15.0:
+        elif self.vs_ver >= 15.0:
             host_dir = (r'bin\HostX86%s' if self.pi.current_is_x86() else
                         r'bin\HostX64%s')
-            tools += [os.path.join(
+            tools += [join(
                 si.VCInstallDir, host_dir % self.pi.target_dir(x64=True))]
 
             if self.pi.current_cpu != self.pi.target_cpu:
-                tools += [os.path.join(
+                tools += [join(
                     si.VCInstallDir, host_dir % self.pi.current_dir(x64=True))]
 
         else:
-            tools += [os.path.join(si.VCInstallDir, 'Bin')]
+            tools += [join(si.VCInstallDir, 'Bin')]
 
         return tools
 
     @property
     def OSLibraries(self):
         """
-        Microsoft Windows SDK Libraries
+        Microsoft Windows SDK Libraries.
+
+        Return
+        ------
+        list of str
+            paths
         """
-        if self.vc_ver <= 10.0:
+        if self.vs_ver <= 10.0:
             arch_subdir = self.pi.target_dir(hidex86=True, x64=True)
-            return [os.path.join(self.si.WindowsSdkDir, 'Lib%s' % arch_subdir)]
+            return [join(self.si.WindowsSdkDir, 'Lib%s' % arch_subdir)]
 
         else:
             arch_subdir = self.pi.target_dir(x64=True)
-            lib = os.path.join(self.si.WindowsSdkDir, 'lib')
+            lib = join(self.si.WindowsSdkDir, 'lib')
             libver = self._sdk_subdir
-            return [os.path.join(lib, '%sum%s' % (libver , arch_subdir))]
+            return [join(lib, '%sum%s' % (libver , arch_subdir))]
 
     @property
     def OSIncludes(self):
         """
-        Microsoft Windows SDK Include
+        Microsoft Windows SDK Include.
+
+        Return
+        ------
+        list of str
+            paths
         """
-        include = os.path.join(self.si.WindowsSdkDir, 'include')
+        include = join(self.si.WindowsSdkDir, 'include')
 
-        if self.vc_ver <= 10.0:
-            return [include, os.path.join(include, 'gl')]
+        if self.vs_ver <= 10.0:
+            return [include, join(include, 'gl')]
 
         else:
-            if self.vc_ver >= 14.0:
+            if self.vs_ver >= 14.0:
                 sdkver = self._sdk_subdir
             else:
                 sdkver = ''
-            return [os.path.join(include, '%sshared' % sdkver),
-                    os.path.join(include, '%sum' % sdkver),
-                    os.path.join(include, '%swinrt' % sdkver)]
+            return [join(include, '%sshared' % sdkver),
+                    join(include, '%sum' % sdkver),
+                    join(include, '%swinrt' % sdkver)]
 
     @property
     def OSLibpath(self):
         """
-        Microsoft Windows SDK Libraries Paths
+        Microsoft Windows SDK Libraries Paths.
+
+        Return
+        ------
+        list of str
+            paths
         """
-        ref = os.path.join(self.si.WindowsSdkDir, 'References')
+        ref = join(self.si.WindowsSdkDir, 'References')
         libpath = []
 
-        if self.vc_ver <= 9.0:
+        if self.vs_ver <= 9.0:
             libpath += self.OSLibraries
 
-        if self.vc_ver >= 11.0:
-            libpath += [os.path.join(ref, r'CommonConfiguration\Neutral')]
+        if self.vs_ver >= 11.0:
+            libpath += [join(ref, r'CommonConfiguration\Neutral')]
 
-        if self.vc_ver >= 14.0:
+        if self.vs_ver >= 14.0:
             libpath += [
                 ref,
-                os.path.join(self.si.WindowsSdkDir, 'UnionMetadata'),
-                os.path.join(
-                    ref,
-                    'Windows.Foundation.UniversalApiContract',
-                    '1.0.0.0',
-                ),
-                os.path.join(
-                    ref,
-                    'Windows.Foundation.FoundationContract',
-                    '1.0.0.0',
-                ),
-                os.path.join(
-                    ref,
-                    'Windows.Networking.Connectivity.WwanContract',
-                    '1.0.0.0',
-                ),
-                os.path.join(
-                    self.si.WindowsSdkDir,
-                    'ExtensionSDKs',
-                    'Microsoft.VCLibs',
-                    '%0.1f' % self.vc_ver,
-                    'References',
-                    'CommonConfiguration',
-                    'neutral',
-                ),
+                join(self.si.WindowsSdkDir, 'UnionMetadata'),
+                join(ref, 'Windows.Foundation.UniversalApiContract', '1.0.0.0'),
+                join(ref, 'Windows.Foundation.FoundationContract', '1.0.0.0'),
+                join(ref,'Windows.Networking.Connectivity.WwanContract',
+                     '1.0.0.0'),
+                join(self.si.WindowsSdkDir, 'ExtensionSDKs', 'Microsoft.VCLibs',
+                     '%0.1f' % self.vs_ver, 'References', 'CommonConfiguration',
+                     'neutral'),
             ]
         return libpath
 
     @property
     def SdkTools(self):
         """
-        Microsoft Windows SDK Tools
+        Microsoft Windows SDK Tools.
+
+        Return
+        ------
+        list of str
+            paths
         """
         return list(self._sdk_tools())
 
     def _sdk_tools(self):
         """
-        Microsoft Windows SDK Tools paths generator
+        Microsoft Windows SDK Tools paths generator.
+
+        Return
+        ------
+        generator of str
+            paths
         """
-        if self.vc_ver < 15.0:
-            bin_dir = 'Bin' if self.vc_ver <= 11.0 else r'Bin\x86'
-            yield os.path.join(self.si.WindowsSdkDir, bin_dir)
+        if self.vs_ver < 15.0:
+            bin_dir = 'Bin' if self.vs_ver <= 11.0 else r'Bin\x86'
+            yield join(self.si.WindowsSdkDir, bin_dir)
 
         if not self.pi.current_is_x86():
             arch_subdir = self.pi.current_dir(x64=True)
             path = 'Bin%s' % arch_subdir
-            yield os.path.join(self.si.WindowsSdkDir, path)
+            yield join(self.si.WindowsSdkDir, path)
 
-        if self.vc_ver == 10.0 or self.vc_ver == 11.0:
+        if self.vs_ver in (10.0, 11.0):
             if self.pi.target_is_x86():
                 arch_subdir = ''
             else:
                 arch_subdir = self.pi.current_dir(hidex86=True, x64=True)
             path = r'Bin\NETFX 4.0 Tools%s' % arch_subdir
-            yield os.path.join(self.si.WindowsSdkDir, path)
+            yield join(self.si.WindowsSdkDir, path)
 
-        elif self.vc_ver >= 15.0:
-            path = os.path.join(self.si.WindowsSdkDir, 'Bin')
+        elif self.vs_ver >= 15.0:
+            path = join(self.si.WindowsSdkDir, 'Bin')
             arch_subdir = self.pi.current_dir(x64=True)
             sdkver = self.si.WindowsSdkLastVersion
-            yield os.path.join(path, '%s%s' % (sdkver, arch_subdir))
+            yield join(path, '%s%s' % (sdkver, arch_subdir))
 
         if self.si.WindowsSDKExecutablePath:
             yield self.si.WindowsSDKExecutablePath
@@ -1059,7 +1342,12 @@
     @property
     def _sdk_subdir(self):
         """
-        Microsoft Windows SDK version subdir
+        Microsoft Windows SDK version subdir.
+
+        Return
+        ------
+        str
+            subdir
         """
         ucrtver = self.si.WindowsSdkLastVersion
         return ('%s\\' % ucrtver) if ucrtver else ''
@@ -1067,22 +1355,32 @@
     @property
     def SdkSetup(self):
         """
-        Microsoft Windows SDK Setup
+        Microsoft Windows SDK Setup.
+
+        Return
+        ------
+        list of str
+            paths
         """
-        if self.vc_ver > 9.0:
+        if self.vs_ver > 9.0:
             return []
 
-        return [os.path.join(self.si.WindowsSdkDir, 'Setup')]
+        return [join(self.si.WindowsSdkDir, 'Setup')]
 
     @property
     def FxTools(self):
         """
-        Microsoft .NET Framework Tools
+        Microsoft .NET Framework Tools.
+
+        Return
+        ------
+        list of str
+            paths
         """
         pi = self.pi
         si = self.si
 
-        if self.vc_ver <= 10.0:
+        if self.vs_ver <= 10.0:
             include32 = True
             include64 = not pi.target_is_x86() and not pi.current_is_x86()
         else:
@@ -1091,102 +1389,142 @@
 
         tools = []
         if include32:
-            tools += [os.path.join(si.FrameworkDir32, ver)
+            tools += [join(si.FrameworkDir32, ver)
                       for ver in si.FrameworkVersion32]
         if include64:
-            tools += [os.path.join(si.FrameworkDir64, ver)
+            tools += [join(si.FrameworkDir64, ver)
                       for ver in si.FrameworkVersion64]
         return tools
 
     @property
     def NetFxSDKLibraries(self):
         """
-        Microsoft .Net Framework SDK Libraries
+        Microsoft .Net Framework SDK Libraries.
+
+        Return
+        ------
+        list of str
+            paths
         """
-        if self.vc_ver < 14.0 or not self.si.NetFxSdkDir:
+        if self.vs_ver < 14.0 or not self.si.NetFxSdkDir:
             return []
 
         arch_subdir = self.pi.target_dir(x64=True)
-        return [os.path.join(self.si.NetFxSdkDir, r'lib\um%s' % arch_subdir)]
+        return [join(self.si.NetFxSdkDir, r'lib\um%s' % arch_subdir)]
 
     @property
     def NetFxSDKIncludes(self):
         """
-        Microsoft .Net Framework SDK Includes
+        Microsoft .Net Framework SDK Includes.
+
+        Return
+        ------
+        list of str
+            paths
         """
-        if self.vc_ver < 14.0 or not self.si.NetFxSdkDir:
+        if self.vs_ver < 14.0 or not self.si.NetFxSdkDir:
             return []
 
-        return [os.path.join(self.si.NetFxSdkDir, r'include\um')]
+        return [join(self.si.NetFxSdkDir, r'include\um')]
 
     @property
     def VsTDb(self):
         """
-        Microsoft Visual Studio Team System Database
+        Microsoft Visual Studio Team System Database.
+
+        Return
+        ------
+        list of str
+            paths
         """
-        return [os.path.join(self.si.VSInstallDir, r'VSTSDB\Deploy')]
+        return [join(self.si.VSInstallDir, r'VSTSDB\Deploy')]
 
     @property
     def MSBuild(self):
         """
-        Microsoft Build Engine
+        Microsoft Build Engine.
+
+        Return
+        ------
+        list of str
+            paths
         """
-        if self.vc_ver < 12.0:
+        if self.vs_ver < 12.0:
             return []
-        elif self.vc_ver < 15.0:
+        elif self.vs_ver < 15.0:
             base_path = self.si.ProgramFilesx86
             arch_subdir = self.pi.current_dir(hidex86=True)
         else:
             base_path = self.si.VSInstallDir
             arch_subdir = ''
 
-        path = r'MSBuild\%0.1f\bin%s' % (self.vc_ver, arch_subdir)
-        build = [os.path.join(base_path, path)]
+        path = r'MSBuild\%0.1f\bin%s' % (self.vs_ver, arch_subdir)
+        build = [join(base_path, path)]
 
-        if self.vc_ver >= 15.0:
+        if self.vs_ver >= 15.0:
             # Add Roslyn C# & Visual Basic Compiler
-            build += [os.path.join(base_path, path, 'Roslyn')]
+            build += [join(base_path, path, 'Roslyn')]
 
         return build
 
     @property
     def HTMLHelpWorkshop(self):
         """
-        Microsoft HTML Help Workshop
+        Microsoft HTML Help Workshop.
+
+        Return
+        ------
+        list of str
+            paths
         """
-        if self.vc_ver < 11.0:
+        if self.vs_ver < 11.0:
             return []
 
-        return [os.path.join(self.si.ProgramFilesx86, 'HTML Help Workshop')]
+        return [join(self.si.ProgramFilesx86, 'HTML Help Workshop')]
 
     @property
     def UCRTLibraries(self):
         """
-        Microsoft Universal C Runtime SDK Libraries
+        Microsoft Universal C Runtime SDK Libraries.
+
+        Return
+        ------
+        list of str
+            paths
         """
-        if self.vc_ver < 14.0:
+        if self.vs_ver < 14.0:
             return []
 
         arch_subdir = self.pi.target_dir(x64=True)
-        lib = os.path.join(self.si.UniversalCRTSdkDir, 'lib')
+        lib = join(self.si.UniversalCRTSdkDir, 'lib')
         ucrtver = self._ucrt_subdir
-        return [os.path.join(lib, '%sucrt%s' % (ucrtver, arch_subdir))]
+        return [join(lib, '%sucrt%s' % (ucrtver, arch_subdir))]
 
     @property
     def UCRTIncludes(self):
         """
-        Microsoft Universal C Runtime SDK Include
+        Microsoft Universal C Runtime SDK Include.
+
+        Return
+        ------
+        list of str
+            paths
         """
-        if self.vc_ver < 14.0:
+        if self.vs_ver < 14.0:
             return []
 
-        include = os.path.join(self.si.UniversalCRTSdkDir, 'include')
-        return [os.path.join(include, '%sucrt' % self._ucrt_subdir)]
+        include = join(self.si.UniversalCRTSdkDir, 'include')
+        return [join(include, '%sucrt' % self._ucrt_subdir)]
 
     @property
     def _ucrt_subdir(self):
         """
-        Microsoft Universal C Runtime SDK version subdir
+        Microsoft Universal C Runtime SDK version subdir.
+
+        Return
+        ------
+        str
+            subdir
         """
         ucrtver = self.si.UniversalCRTSdkLastVersion
         return ('%s\\' % ucrtver) if ucrtver else ''
@@ -1194,31 +1532,52 @@
     @property
     def FSharp(self):
         """
-        Microsoft Visual F#
+        Microsoft Visual F#.
+
+        Return
+        ------
+        list of str
+            paths
         """
-        if self.vc_ver < 11.0 and self.vc_ver > 12.0:
+        if 11.0 > self.vs_ver > 12.0:
             return []
 
-        return self.si.FSharpInstallDir
+        return [self.si.FSharpInstallDir]
 
     @property
     def VCRuntimeRedist(self):
         """
-        Microsoft Visual C++ runtime redistribuable dll
+        Microsoft Visual C++ runtime redistributable dll.
+
+        Return
+        ------
+        str
+            path
         """
-        arch_subdir = self.pi.target_dir(x64=True)
-        if self.vc_ver < 15:
-            redist_path = self.si.VCInstallDir
-            vcruntime = 'redist%s\\Microsoft.VC%d0.CRT\\vcruntime%d0.dll'
-        else:
-            redist_path = self.si.VCInstallDir.replace('\\Tools', '\\Redist')
-            vcruntime = 'onecore%s\\Microsoft.VC%d0.CRT\\vcruntime%d0.dll'
+        vcruntime = 'vcruntime%d0.dll' % self.vc_ver
+        arch_subdir = self.pi.target_dir(x64=True).strip('\\')
 
-        # Visual Studio 2017  is still Visual C++ 14.0
-        dll_ver = 14.0 if self.vc_ver == 15 else self.vc_ver
+        # Installation prefixes candidates
+        prefixes = []
+        tools_path = self.si.VCInstallDir
+        redist_path = dirname(tools_path.replace(r'\Tools', r'\Redist'))
+        if isdir(redist_path):
+            # Redist version may not be exactly the same as tools
+            redist_path = join(redist_path, listdir(redist_path)[-1])
+            prefixes += [redist_path, join(redist_path, 'onecore')]
 
-        vcruntime = vcruntime % (arch_subdir, self.vc_ver, dll_ver)
-        return os.path.join(redist_path, vcruntime)
+        prefixes += [join(tools_path, 'redist')]  # VS14 legacy path
+
+        # CRT directory
+        crt_dirs = ('Microsoft.VC%d.CRT' % (self.vc_ver * 10),
+                    # Sometime store in directory with VS version instead of VC
+                    'Microsoft.VC%d.CRT' % (int(self.vs_ver) * 10))
+
+        # vcruntime path
+        for prefix, crt_dir in itertools.product(prefixes, crt_dirs):
+            path = join(prefix, arch_subdir, crt_dir, vcruntime)
+            if isfile(path):
+                return path
 
     def return_env(self, exists=True):
         """
@@ -1228,6 +1587,11 @@
         ----------
         exists: bool
             It True, only return existing paths.
+
+        Return
+        ------
+        dict
+            environment
         """
         env = dict(
             include=self._build_paths('include',
@@ -1261,7 +1625,7 @@
                                     self.FSharp],
                                    exists),
         )
-        if self.vc_ver >= 14 and os.path.isfile(self.VCRuntimeRedist):
+        if self.vs_ver >= 14 and isfile(self.VCRuntimeRedist):
             env['py_vcruntime_redist'] = self.VCRuntimeRedist
         return env
 
@@ -1272,20 +1636,35 @@
         unique, extant, directories from those paths and from
         the environment variable. Raise an error if no paths
         are resolved.
+
+        Parameters
+        ----------
+        name: str
+            Environment variable name
+        spec_path_lists: list of str
+            Paths
+        exists: bool
+            It True, only return existing paths.
+
+        Return
+        ------
+        str
+            Pathsep-separated paths
         """
         # flatten spec_path_lists
         spec_paths = itertools.chain.from_iterable(spec_path_lists)
-        env_paths = safe_env.get(name, '').split(os.pathsep)
+        env_paths = environ.get(name, '').split(pathsep)
         paths = itertools.chain(spec_paths, env_paths)
-        extant_paths = list(filter(os.path.isdir, paths)) if exists else paths
+        extant_paths = list(filter(isdir, paths)) if exists else paths
         if not extant_paths:
             msg = "%s environment variable is empty" % name.upper()
             raise distutils.errors.DistutilsPlatformError(msg)
         unique_paths = self._unique_everseen(extant_paths)
-        return os.pathsep.join(unique_paths)
+        return pathsep.join(unique_paths)
 
     # from Python docs
-    def _unique_everseen(self, iterable, key=None):
+    @staticmethod
+    def _unique_everseen(iterable, key=None):
         """
         List unique elements, preserving order.
         Remember all elements ever seen.
diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py
--- a/rpython/translator/platform/windows.py
+++ b/rpython/translator/platform/windows.py
@@ -5,6 +5,7 @@
 from rpython.translator.platform import CompilationError
 from rpython.translator.platform import log, _run_subprocess
 from rpython.translator.platform import Platform, posix
+import rpython.tool.setuptools_msvc as msvc
 
 import rpython
 rpydir = str(py.path.local(rpython.__file__).join('..'))
@@ -37,96 +38,37 @@
     return _get_compiler_type(cc, True)
 
 def _find_vcvarsall(version, x64flag):
-    import rpython.tool.setuptools_msvc as msvc
     if x64flag:
         arch = 'x64'
     else:
         arch = 'x86'
-    if version >= 140:
-        return msvc.msvc14_get_vc_env(arch)
-    else:
-        return msvc.msvc9_query_vcvarsall(version / 10.0, arch)
+    return msvc.EnvironmentInfo(arch, vc_ver=version/10.0).return_env()
 
-def _get_msvc_env(vsver, x64flag):
+def _get_msvc_env(vcver, x64flag):
     vcdict = None
     toolsdir = None
+    # use setuptools from python3 to find tools
     try:
-        toolsdir = os.environ['VS%sCOMNTOOLS' % vsver]
-    except KeyError:
-        # use setuptools from python3 to find tools
-        try:
-            vcdict = _find_vcvarsall(vsver, x64flag)
-        except ImportError as e:
-            if 'setuptools' in str(e):
-                log.error('is setuptools installed (perhaps try %s -mensurepip)?' % sys.executable)
-            log.error('looking for compiler %s raised exception "%s' % (vsver, str(e)))
-        except Exception as e:
-            log.error('looking for compiler %s raised exception "%s' % (vsver, str(e)))
+        available = msvc.EnvironmentInfo('x86').si.find_reg_vs_vers()
+        if vcver / 10.0 not in available:
+            log.error('Could not find vcver %i' % vcver)
             return None
-    else:
-        if x64flag:
-            vsinstalldir = os.path.abspath(os.path.join(toolsdir, '..', '..'))
-            vcinstalldir = os.path.join(vsinstalldir, 'VC')
-            vcbindir = os.path.join(vcinstalldir, 'BIN')
-            vcvars = os.path.join(vcbindir, 'amd64', 'vcvarsamd64.bat')
-        else:
-            vcvars = os.path.join(toolsdir, 'vsvars32.bat')
-            if not os.path.exists(vcvars):
-                # even msdn does not know which to run
-                # see https://msdn.microsoft.com/en-us/library/1700bbwd(v=vs.90).aspx
-                # which names both
-                vcvars = os.path.join(toolsdir, 'vcvars32.bat')
-
-        import subprocess
-        try:
-            popen = subprocess.Popen('"%s" & set' % (vcvars,),
-                                 stdout=subprocess.PIPE,
-                                 stderr=subprocess.PIPE)
-
-            stdout, stderr = popen.communicate()
-            if popen.wait() != 0 or stdout[:5].lower() == 'error':
-                log.msg('Running "%s" errored: \n\nstdout:\n%s\n\nstderr:\n%s' % (
-                    vcvars, stdout.split()[0], stderr))
-                return None
-            else:
-                log.msg('Running "%s" succeeded' %(vcvars,))
-        except Exception as e:
-            log.msg('Running "%s" failed: "%s"' % (vcvars, str(e)))
-            return None
-
-        stdout = stdout.replace("\r\n", "\n")
-        vcdict = {}
-        for line in stdout.split("\n"):
-            if '=' not in line:
-                continue
-            key, value = line.split('=', 1)
-            vcdict[key] = value
-    env = {}
-    for key, value in vcdict.items():
-        if key.upper() in ['PATH', 'INCLUDE', 'LIB']:
-            env[key.upper()] = value
-    if 'PATH' not in env:
-        log.msg('Did not find "PATH" in stdout\n%s' %(stdout))
-    if not _find_executable('mt.exe', env['PATH']):
-        # For some reason the sdk bin path is missing?
-        # put it together from some other env variables that happened to exist
-        # on the buildbot where this occurred
-        if 'WindowsSDKVersion' in vcdict and 'WindowsSdkDir' in vcdict:
-            binpath = vcdict['WindowsSdkDir'] + '\\bin\\' + vcdict['WindowsSDKVersion'] + 'x86'
-            env['PATH'] += ';' + binpath
-        if not _find_executable('mt.exe', env['PATH']):
-            log.msg('Could not find mt.exe on path=%s' % env['PATH'])
-            log.msg('Running vsver %s set this env' % vsver)
-            for key, value in vcdict.items():
-                log.msg('%s=%s' %(key, value))
-    log.msg("Updated environment with vsver %d, using x64 %s" % (vsver, x64flag,))
-    return env
+        vcdict = _find_vcvarsall(vcver, x64flag)
+        keys = vcdict.keys()
+        for k in keys:
+            if k.upper != k:
+                vcdict[k.upper()] = vcdict[k]
+                del vcdict[k]
+        log.error('Found vcver %i' % vcver)
+        return vcdict
+    except Exception as e:
+        log.error('looking for compiler %s raised exception "%s' % (vcver, str(e)))
+        return None
 
 def find_msvc_env(x64flag=False, ver0=None):
-    vcvers = [140, 141, 150, 90, 100]
+    vcvers = [150, 140, 141, 90, 100]
     if ver0 in vcvers:
         vcvers.insert(0, ver0)
-    errs = []
     for vsver in vcvers:
         env = _get_msvc_env(vsver, x64flag)
         if env is not None:

From pypy.commits at gmail.com  Sun Nov 10 18:16:39 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 10 Nov 2019 15:16:39 -0800 (PST)
Subject: [pypy-commit] pypy code_page-utf8: start to implement code_page and
 oem encoder/decoder
Message-ID: <5dc89a57.1c69fb81.ee299.5ffb@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: code_page-utf8
Changeset: r98013:a55b2a60b689
Date: 2019-11-09 01:36 +0200
http://bitbucket.org/pypy/pypy/changeset/a55b2a60b689/

Log:	start to implement code_page and oem encoder/decoder

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -374,6 +374,18 @@
         res_utf8 = runicode.unicode_encode_utf_8(res, size, 'strict')
         return res_utf8, len(res), size
 
+    def utf8_encode_code_page(s, errors, errorhandler, allow_surrogates=False):
+        pass
+
+    def str_decode_code_page(s, errors, final, errorhandler, force_ignore=True):
+        pass
+
+    def utf8_encode_oem(s, errors, errorhandler, allow_surrogates=False):
+        pass
+
+    def str_decode_oem(s, errors, final, errorhandler, force_ignore=True):
+        pass
+
 def str_decode_utf8(s, errors, final, errorhandler, allow_surrogates=False):
     try:
         # fast version first
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -3,7 +3,7 @@
 from rpython.rlib.objectmodel import we_are_translated, not_rpython
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 from rpython.rlib.rutf8 import MAXUNICODE
-from rpython.rlib.runicode import raw_unicode_escape_helper
+from rpython.rlib import runicode
 
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
@@ -291,7 +291,7 @@
         obj = w_obj._utf8
         while pos < end:
             code = rutf8.codepoint_at_pos(obj, pos)
-            raw_unicode_escape_helper(builder, code)
+            unicodehelper.raw_unicode_escape_helper(builder, code)
             pos = rutf8.next_codepoint_pos(obj, pos)
         return space.newtuple([space.newtext(builder.build()), w_end])
     elif space.isinstance_w(w_exc, space.w_UnicodeDecodeError):
@@ -303,7 +303,7 @@
         pos = start
         while pos < end:
             oc = ord(obj[pos])
-            raw_unicode_escape_helper(builder, oc)
+            unicodehelper.raw_unicode_escape_helper(builder, oc)
             pos += 1
         return space.newtuple([space.newtext(builder.build()), w_end])
     else:
@@ -705,10 +705,13 @@
          ]:
     make_decoder_wrapper(decoder)
 
-from rpython.rlib import runicode
-if hasattr(runicode, 'str_decode_mbcs'):
+if getattr(unicodehelper, '_WIN32', False):
     make_encoder_wrapper('mbcs_encode')
     make_decoder_wrapper('mbcs_decode')
+    make_encoder_wrapper('code_page_encode')
+    make_decoder_wrapper('code_page_decode')
+    make_encoder_wrapper('oem_encode')
+    make_decoder_wrapper('oem_decode')
 
 # utf-8 functions are not regular, because we have to pass
 # "allow_surrogates=False"
diff --git a/pypy/module/_codecs/moduledef.py b/pypy/module/_codecs/moduledef.py
--- a/pypy/module/_codecs/moduledef.py
+++ b/pypy/module/_codecs/moduledef.py
@@ -91,7 +91,11 @@
         from rpython.rlib import runicode
         if (hasattr(runicode, 'str_decode_mbcs')):
             self.interpleveldefs['mbcs_encode'] = 'interp_codecs.mbcs_encode'
+            self.interpleveldefs['oem_encode'] = 'interp_codecs.oem_encode'
+            self.interpleveldefs['code_page_encode'] = 'interp_codecs.code_page_encode'
             self.interpleveldefs['mbcs_decode'] = 'interp_codecs.mbcs_decode'
+            self.interpleveldefs['oem_decode'] = 'interp_codecs.oem_decode'
+            self.interpleveldefs['code_page_decode'] = 'interp_codecs.code_page_decode'
 
         MixedModule.__init__(self, space, *args)
 

From pypy.commits at gmail.com  Sun Nov 10 18:16:40 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 10 Nov 2019 15:16:40 -0800 (PST)
Subject: [pypy-commit] pypy code_page-utf8: test,
 implement code page encoding/decoding via a new
 unicodehelper_win32.py
Message-ID: <5dc89a58.1c69fb81.d6c09.fb0d@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: code_page-utf8
Changeset: r98014:b27e7cddb1e0
Date: 2019-11-10 14:34 -0500
http://bitbucket.org/pypy/pypy/changeset/b27e7cddb1e0/

Log:	test, implement code page encoding/decoding via a new
	unicodehelper_win32.py

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -61,11 +61,10 @@
     state = space.fromcache(interp_codecs.CodecState)
     errorhandler=state.decode_error_handler
     if _WIN32:
+        import pypy.interpreter.unicodehelper_win32 as win32
         bytes = space.bytes_w(w_string)
         slen = len(bytes)
-        uni, lgt = runicode.str_decode_mbcs(bytes, slen, 'strict', final=True,
-                           errorhandler=errorhandler, force_ignore=False)
-        utf8 = uni.encode('utf-8')
+        utf8, _, lgt = str_decode_mbcs(bytes, 'strict', True, errorhandler)
     elif 0 and  _MACOSX:
         bytes = space.bytes_w(w_string)
         utf8, lgt, pos  = str_decode_utf8(bytes, 'surrogateescape', True,
@@ -362,29 +361,31 @@
     return result.build()
 
 if _WIN32:
+    import pypy.interpreter.unicodehelper_win32 as win32
     def utf8_encode_mbcs(s, errors, errorhandler, allow_surrogates=False):
-        res = rutf8.utf8_encode_mbcs(s, errors, errorhandler,
-                                     force_replace=False)
+        res = win32.utf8_encode_mbcs(s, errors, errorhandler)
         return res
 
-    def str_decode_mbcs(s, errors, final, errorhandler, force_ignore=True):
-        slen = len(s)
-        res, size = runicode.str_decode_mbcs(s, slen, errors, final=final,
-                           errorhandler=errorhandler, force_ignore=force_ignore)
-        res_utf8 = runicode.unicode_encode_utf_8(res, size, 'strict')
-        return res_utf8, len(res), size
-
-    def utf8_encode_code_page(s, errors, errorhandler, allow_surrogates=False):
-        pass
-
-    def str_decode_code_page(s, errors, final, errorhandler, force_ignore=True):
-        pass
+    def str_decode_mbcs(s, errors, final, errorhandler):
+        res, size = win32.str_decode_mbcs(s, errors, errorhandler, final=final)
+        return res, len(res), size
 
     def utf8_encode_oem(s, errors, errorhandler, allow_surrogates=False):
-        pass
+        res = win32.utf8_encode_oem(s, errors, errorhandler)
+        return res
 
-    def str_decode_oem(s, errors, final, errorhandler, force_ignore=True):
-        pass
+    def str_decode_oem(s, errors, final, errorhandler):
+        res, size = win32.str_decode_oem(s, errors, errorhandler, final)
+        return res, len(res), size
+
+    def utf8_encode_code_page(cp, s, errors, errorhandler, allow_surrogates=False):
+        res = win32.utf8_encode_code_page(cp, s, errors, errorhandler)
+        return res
+
+    def str_decode_code_page(cp, s, errors, final, errorhandler):
+        res, size = win32.str_decode_code_page(cp, s, errors, errorhandler, final)
+        return res, len(res), size
+
 
 def str_decode_utf8(s, errors, final, errorhandler, allow_surrogates=False):
     try:
diff --git a/pypy/interpreter/unicodehelper_win32.py b/pypy/interpreter/unicodehelper_win32.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/unicodehelper_win32.py
@@ -0,0 +1,202 @@
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rlib.runicode import (BOOLP, WideCharToMultiByte,
+         MultiByteToWideChar)
+from rpython.rlib.rutf8 import (Utf8StringBuilder, Utf8StringIterator,
+         next_codepoint_pos)
+from rpython.rlib import rwin32
+
+def Py_UNICODE_HIGH_SURROGATE(ch):
+   return rffi.cast(lltype.UniChar, 0xD800 - (0x10000 >> 10) + ((ch) >> 10))
+
+def Py_UNICODE_LOW_SURROGATE(ch):
+    return rffi.cast(lltype.uniChar, 0xDC00 + ((ch) & 0x3FF))
+
+if rffi.sizeof(rffi.INT) < rffi.sizeof(rffi.SIZE_T):
+    NEED_RETRY = True
+else:
+    NEED_RETRY = False
+WC_ERR_INVALID_CHARS = 0x0080
+
+code_page_map = {
+        rwin32.CP_ACP: "mbcs",
+        rwin32.CP_UTF7:"CP_UTF7",
+        rwin32.CP_UTF8:"CP_UTF8",
+    }
+
+def _code_page_name(code_page):
+    return code_page_map.get(code_page, "cp%d" % code_page)
+
+def _decode_code_page_flags(code_page):
+    if code_page == rwin32.CP_UTF7:
+        # The CP_UTF7 decoder only supports flags==0
+        return 0
+    return rwin32.MB_ERR_INVALID_CHARS
+
+def _encode_code_page_flags(code_page, errors):
+    if code_page == rwin32.CP_UTF8:
+        return WC_ERR_INVALID_CHARS
+    elif code_page == rwin32.CP_UTF7:
+        return 0
+    if errors == 'replace':
+        return 0
+    return rwin32.WC_NO_BEST_FIT_CHARS
+
+def _decode_cp_error(s, errorhandler, encoding, errors, start, end):
+    # late import to avoid circular import
+    from pypy.interpreter.unicodehelper import _str_decode_utf8_slowpath
+    if rwin32.GetLastError_saved() == rwin32.ERROR_NO_UNICODE_TRANSLATION:
+        msg = ("No mapping for the Unicode character exists in the target "
+               "multi-byte code page.")
+        r, ignore1, ignore2 = _str_decode_utf8_slowpath(s[start:end], errors, False, errorhandler, False)
+        return r, end
+    else:
+        raise rwin32.lastSavedWindowsError()
+
+def _unibuf_to_utf8(uni, insize):
+    """Encode the widechar unicode buffer u to utf8
+    Should never error, since the buffer comes from a call to
+    MultiByteToWideChar
+    """
+    flags = 0
+    cp = rwin32.CP_UTF8
+    used_default_p = lltype.nullptr(BOOLP.TO)
+    assert uni is not None
+    with rffi.scoped_nonmoving_unicodebuffer(uni) as dataptr:
+        # first get the size of the result
+        outsize = WideCharToMultiByte(cp, flags, dataptr, insize,
+                                    None, 0, None, used_default_p)
+        if outsize == 0:
+            raise rwin32.lastSavedWindowsError()
+        with rffi.scoped_alloc_buffer(outsize) as buf:
+            # do the conversion
+            if WideCharToMultiByte(cp, flags, dataptr, insize, buf.raw,
+                    outsize, None, used_default_p) == 0:
+                raise rwin32.lastSavedWindowsError()
+            result = buf.str(outsize)
+            assert result is not None
+            return result
+
+def _decode_helper(cp, s, flags, encoding, errors, errorhandler, 
+                   start, end, res):
+    if end > len(s):
+        end = len(s)
+    piece = s[start:end]
+    with rffi.scoped_nonmovingbuffer(piece) as dataptr:
+        # first get the size of the result
+        outsize = MultiByteToWideChar(cp, flags, dataptr, len(piece),
+                                    lltype.nullptr(rffi.CWCHARP.TO), 0)
+        if outsize == 0:
+            r, pos = _decode_cp_error(s, errorhandler,
+                                               encoding, errors, start, end)
+            res.append(r)
+            return pos
+
+        with rffi.scoped_alloc_unicodebuffer(outsize) as buf:
+            # do the conversion
+            if MultiByteToWideChar(cp, flags, dataptr, len(piece),
+                                   buf.raw, outsize) == 0:
+                r, pos = _decode_cp_error(s, errorhandler,
+                                             encoding, errors, start, end)
+                res.append(r)
+                return pos
+            else:
+                res.append(_unibuf_to_utf8(buf.str(outsize), outsize))
+    return end
+
+def str_decode_code_page(cp, s, errors, errorhandler, final=False):
+    """Decodes a byte string s from a code page cp with an error handler.
+    Returns utf8 result, original s length
+    """
+    insize = len(s)
+    if insize == 0:
+        return '', 0
+    flags = _decode_code_page_flags(cp)
+    encoding = _code_page_name(cp)
+    assert errorhandler is not None
+    res = Utf8StringBuilder(insize)
+    if errors == 'strict':
+        _decode_helper(cp, s, flags, encoding, errors, errorhandler,
+                       0, len(s), res)
+    else:
+        prev_pos = 0
+        pos = 0
+        while pos < len(s):
+            pos = next_codepoint_pos(s, prev_pos)
+            pos = _decode_helper(cp, s, flags, encoding,
+                            errors, errorhandler, prev_pos, pos, res)
+            prev_pos = pos
+    return res.build(), insize
+
+def str_decode_mbcs(s, errors, errorhandler, final=False):
+    return str_decode_code_page(rwin32.CP_ACP, s, errors, errorhandler, final)
+
+def str_decode_oem(s, errors, errorhandler, final=False):
+    return str_decode_code_page(rwin32.CP_OEMCP, s, errors, errorhandler, final)
+
+def utf8_encode_code_page(cp, s, errors, errorhandler):
+    """Encode a utf8 string s using code page cp and the given
+    errors/errorhandler.
+    Returns a encoded byte string
+    """
+
+    name = _code_page_name(cp)
+    lgt = len(s)
+
+    if lgt == 0:
+        return ''
+    flags = _encode_code_page_flags(cp, errors)
+    if cp in (rwin32.CP_UTF8, rwin32.CP_UTF7):
+        used_default_p = lltype.nullptr(BOOLP.TO)
+    else:
+        used_default_p = lltype.malloc(BOOLP.TO, 1, flavor='raw')
+    # Encode one codpoint at a time to allow the errorhandlers to do
+    # their thing
+    chars = lltype.malloc(rffi.CWCHARP.TO, 2, flavor = 'raw')
+    res = Utf8StringBuilder(lgt)
+    try:
+        pos = 0
+        for uni in Utf8StringIterator(s):
+            if used_default_p:
+                used_default_p[0] = rffi.cast(rwin32.BOOL, False)
+            if uni < 0x10000:
+                chars[0] = rffi.cast(lltype.UniChar, uni)
+                charsize = 1
+            else:
+                chars[0] = Py_UNICODE_HIGH_SURROGATE(uni)
+                chars[0] = Py_UNICODE_LOW_SURROGATE(uni)
+                charsize = 2
+                # first get the size of the result
+            outsize = WideCharToMultiByte(cp, flags, chars, charsize, None, 0,
+                                           None, used_default_p)
+            if outsize == 0:
+                raise rwin32.lastSavedWindowsError()
+            # If we used a default char, then we failed!
+            if (used_default_p and rffi.cast(lltype.Bool, used_default_p[0])):
+                r, pos, retype = errorhandler(errors, name, "invalid character", s, pos, pos+1)
+                res.append(r)
+                continue
+            with rffi.scoped_alloc_buffer(outsize) as buf:
+                # do the conversion
+                if WideCharToMultiByte(cp, flags,
+                                       chars, charsize, buf.raw, outsize,
+                                       None, used_default_p) == 0:
+                    raise rwin32.lastSavedWindowsError()
+                if (used_default_p and
+                    rffi.cast(lltype.Bool, used_default_p[0])):
+                        r, pos, rettype = errorhandler(errors, name, "invalid character",
+                                           s, pos, pos + 1)
+                        res.append(r)
+                else:
+                    res.append(buf.str(outsize))
+            pos += 1
+        return res.build()
+    finally:
+        lltype.free(chars, flavor='raw')
+        if used_default_p:
+            lltype.free(used_default_p, flavor='raw')
+
+def utf8_encode_mbcs(s, errors, errorhandler):
+        return utf8_encode_code_page(rwin32.CP_ACP, s, errors, errorhandler)
+
+def utf8_encode_oem(s, errors, errorhandler):
+        return utf8_encode_code_page(rwin32.CP_OEMCP, s, errors, errorhandler)
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -708,11 +708,41 @@
 if getattr(unicodehelper, '_WIN32', False):
     make_encoder_wrapper('mbcs_encode')
     make_decoder_wrapper('mbcs_decode')
-    make_encoder_wrapper('code_page_encode')
-    make_decoder_wrapper('code_page_decode')
     make_encoder_wrapper('oem_encode')
     make_decoder_wrapper('oem_decode')
 
+    # need to add the code_page argument
+
+    @unwrap_spec(code_page=int, errors='text_or_none')
+    def code_page_encode(space, code_page, w_arg, errors="strict"):
+        # w_arg is a W_Unicode or W_Bytes?
+        w_arg = space.convert_arg_to_w_unicode(w_arg, errors)
+        if errors is None:
+            errors = 'strict'
+        allow_surrogates = False
+        if errors in ('surrogatepass',):
+            allow_surrogates = True
+        state = space.fromcache(CodecState)
+        ulen = w_arg._length
+        result = unicodehelper.utf8_encode_code_page(code_page, w_arg._utf8,
+                      errors, state.encode_error_handler,
+                      allow_surrogates=allow_surrogates)
+        return space.newtuple([space.newbytes(result), space.newint(ulen)])
+
+    @unwrap_spec(code_page=int, string='bufferstr', errors='text_or_none',
+                 w_final=WrappedDefault(False))
+    def code_page_decode(space, code_page, string, errors="strict", w_final=None):
+        if errors is None:
+            errors = 'strict'
+        final = space.is_true(w_final)
+        state = space.fromcache(CodecState)
+        result, length, pos = unicodehelper.str_decode_code_page(code_page,
+                                   string, errors, final,
+                                   state.decode_error_handler)
+        # must return bytes, pos
+        return space.newtuple([space.newutf8(result, length), space.newint(pos)])
+
+
 # utf-8 functions are not regular, because we have to pass
 # "allow_surrogates=False"
 @unwrap_spec(errors='text_or_none')
diff --git a/pypy/module/_codecs/moduledef.py b/pypy/module/_codecs/moduledef.py
--- a/pypy/module/_codecs/moduledef.py
+++ b/pypy/module/_codecs/moduledef.py
@@ -1,5 +1,6 @@
 from pypy.interpreter.mixedmodule import MixedModule
 from rpython.rlib.objectmodel import not_rpython
+from rpython.rlib import rwin32
 from pypy.module._codecs import interp_codecs
 
 class Module(MixedModule):
@@ -87,9 +88,8 @@
 
     @not_rpython
     def __init__(self, space, *args):
-        # mbcs codec is Windows specific, and based on rffi.
-        from rpython.rlib import runicode
-        if (hasattr(runicode, 'str_decode_mbcs')):
+        # mbcs codec is Windows specific, and based on rffi system calls.
+        if rwin32.WIN32:
             self.interpleveldefs['mbcs_encode'] = 'interp_codecs.mbcs_encode'
             self.interpleveldefs['oem_encode'] = 'interp_codecs.oem_encode'
             self.interpleveldefs['code_page_encode'] = 'interp_codecs.code_page_encode'
diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -432,6 +432,100 @@
             assert((b'aaaa' + seq + b'bbbb').decode('utf-8', 'ignore') ==
                           'aaaa' + res + 'bbbb')
 
+ at pytest.mark.skipif(sys.platform != 'win32', reason='win32-only')
+class AppTestCodePage:
+    spaceconfig = {
+    }
+
+    def test_code_pages(self):
+        import _codecs as codecs
+        def check_decode(cp, test):
+            raw, errors, expected  = test
+            if expected is not None:
+                try:
+                    decoded = codecs.code_page_decode(cp, raw, errors, True)
+                except UnicodeDecodeError as err:
+                    assert False, ('Unable to decode %a from "cp%s" with '
+                              'errors=%r: %s' % (raw, cp, errors, err))
+                assert decoded[0] == expected, ('%a.decode("cp%s", %r)=%a != %a'
+                    % (raw, cp, errors, decoded[0], expected))
+                assert decoded[1] >= 0
+                assert decoded[1] <= len(raw)
+            else:
+                raises(UnicodeDecodeError,
+                    codecs.code_page_decode, cp, raw, errors, True)
+
+        def check_encode(cp, test):
+            text, errors, expected = test
+            if expected is not None:
+                try:
+                    encoded = codecs.code_page_encode(cp, text, errors)
+                except UnicodeEncodeError as err:
+                    assert False, ('Unable to encode %a to "cp%s" with '
+                              'errors=%r: %s' % (text, cp, errors, err))
+                assert encoded[0] == expected, ('%a.encode("cp%s", %r)=%a != %a'
+                    % (text, cp, errors, encoded[0], expected))
+                assert encoded[1] == len(text)
+            else:
+                raises(UnicodeEncodeError,
+                    codecs.code_page_encode, cp, text, errors)
+
+        for test in (
+                (u'abc', 'strict', b'abc'),
+                (u'\uff44\u9a3e', 'strict', b'\x82\x84\xe9\x80'),
+                # test error handlers
+                (u'\xff', 'strict', None),
+                (u'[\xff]', 'ignore', b'[]'),
+                (u'[\xff]', 'replace', b'[y]'),
+                (u'[\u20ac]', 'replace', b'[?]'),
+                (u'[\xff]', 'backslashreplace', b'[\\xff]'),
+                (u'[\xff]', 'namereplace',
+                 b'[\\N{LATIN SMALL LETTER Y WITH DIAERESIS}]'),
+                (u'[\xff]', 'xmlcharrefreplace', b'[&#255;]'),
+                (u'\udcff', 'strict', None),
+                (u'[\udcff]', 'surrogateescape', b'[\xff]'),
+                (u'[\udcff]', 'surrogatepass', None),
+            ):
+            check_encode(932, test)
+
+        for test in (
+                (b'abc', 'strict', u'abc'),
+                (b'\x82\x84\xe9\x80', 'strict', u'\uff44\u9a3e'),
+                # invalid bytes
+                (b'[\xff]', 'strict', None),
+                (b'[\xff]', 'ignore', u'[]'),
+                (b'[\xff]', 'replace', u'[\ufffd]'),
+                (b'[\xff]', 'backslashreplace', u'[\\xff]'),
+                (b'[\xff]', 'surrogateescape', u'[\udcff]'),
+                (b'[\xff]', 'surrogatepass', None),
+                (b'\x81\x00abc', 'strict', None),
+                (b'\x81\x00abc', 'ignore', u'\x00abc'),
+                (b'\x81\x00abc', 'replace', u'\ufffd\x00abc'),
+                (b'\x81\x00abc', 'backslashreplace', u'\\x81\x00abc'),
+            ):
+            check_decode(932, test)
+
+        for test in (
+                (u'abc', 'strict', b'abc'),
+                (u'\xe9\u20ac', 'strict',  b'\xe9\x80'),
+                (u'\xff', 'strict', b'\xff'),
+                # test error handlers
+                (u'\u0141', 'strict', None),
+                (u'\u0141', 'ignore', b''),
+                (u'\u0141', 'replace', b'L'),
+                (u'\udc98', 'surrogateescape', b'\x98'),
+                (u'\udc98', 'surrogatepass', None),
+            ):
+            check_encode(1252, test)
+
+        for test in (
+                (b'abc', 'strict', u'abc'),
+                (b'\xe9\x80', 'strict', u'\xe9\u20ac'),
+                (b'\xff', 'strict', u'\xff'),
+            ):
+            check_decode(1252, test)
+
+
 
 class AppTestPartialEvaluation:
     spaceconfig = dict(usemodules=['array',])
diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py
--- a/rpython/rlib/rwin32.py
+++ b/rpython/rlib/rwin32.py
@@ -114,6 +114,7 @@
                        WC_NO_BEST_FIT_CHARS STD_INPUT_HANDLE STD_OUTPUT_HANDLE
                        STD_ERROR_HANDLE HANDLE_FLAG_INHERIT FILE_TYPE_CHAR
                        LOAD_WITH_ALTERED_SEARCH_PATH
+                       CP_ACP CP_UTF8 CP_UTF7 CP_OEMCP MB_ERR_INVALID_CHARS
                     """
         from rpython.translator.platform import host_factory
         static_platform = host_factory()

From pypy.commits at gmail.com  Sun Nov 10 18:16:42 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 10 Nov 2019 15:16:42 -0800 (PST)
Subject: [pypy-commit] pypy py3.6: merge default into py3.6
Message-ID: <5dc89a5a.1c69fb81.e6994.2818@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r98015:e01187956256
Date: 2019-11-10 18:12 -0500
http://bitbucket.org/pypy/pypy/changeset/e01187956256/

Log:	merge default into py3.6

diff --git a/extra_tests/cffi_tests/cffi0/test_verify.py b/extra_tests/cffi_tests/cffi0/test_verify.py
--- a/extra_tests/cffi_tests/cffi0/test_verify.py
+++ b/extra_tests/cffi_tests/cffi0/test_verify.py
@@ -4,6 +4,7 @@
 import sys, os, math, weakref
 from cffi import FFI, VerificationError, VerificationMissing, model, FFIError
 from extra_tests.cffi_tests.support import *
+from extra_tests.cffi_tests.support import extra_compile_args
 
 
 lib_m = ['m']
@@ -14,17 +15,6 @@
         lib_m = ['msvcrt']
     pass      # no obvious -Werror equivalent on MSVC
 else:
-    if (sys.platform == 'darwin' and
-          [int(x) for x in os.uname()[2].split('.')] >= [11, 0, 0]):
-        # assume a standard clang or gcc
-        extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion']
-        # special things for clang
-        extra_compile_args.append('-Qunused-arguments')
-    else:
-        # assume a standard gcc
-        extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion',
-                              '-Wno-unused-parameter']
-
     class FFI(FFI):
         def verify(self, *args, **kwds):
             return super(FFI, self).verify(
diff --git a/extra_tests/cffi_tests/cffi1/test_recompiler.py b/extra_tests/cffi_tests/cffi1/test_recompiler.py
--- a/extra_tests/cffi_tests/cffi1/test_recompiler.py
+++ b/extra_tests/cffi_tests/cffi1/test_recompiler.py
@@ -35,8 +35,9 @@
         source = 'extern "C" {\n%s\n}' % (source,)
     elif sys.platform != 'win32':
         # add '-Werror' to the existing 'extra_compile_args' flags
+        from extra_tests.cffi_tests.support import extra_compile_args
         kwds['extra_compile_args'] = (kwds.get('extra_compile_args', []) +
-                                      ['-Werror'])
+                                      extra_compile_args)
     return _verify(ffi, module_name, source, *args, **kwds)
 
 def test_set_source_no_slashes():
@@ -2039,7 +2040,7 @@
     ffi.cdef("float _Complex f1(float a, float b);");
     lib = verify(ffi, "test_function_returns_float_complex", """
         #include <complex.h>
-        static float _Complex f1(float a, float b) { return a + I*2.0*b; }
+        static float _Complex f1(float a, float b) { return a + I*2.0f*b; }
     """, no_cpp=True)    # <complex.h> fails on some systems with C++
     result = lib.f1(1.25, 5.1)
     assert type(result) == complex
diff --git a/extra_tests/cffi_tests/cffi1/test_verify1.py b/extra_tests/cffi_tests/cffi1/test_verify1.py
--- a/extra_tests/cffi_tests/cffi1/test_verify1.py
+++ b/extra_tests/cffi_tests/cffi1/test_verify1.py
@@ -5,7 +5,7 @@
 from cffi import CDefError
 from cffi import recompiler
 from extra_tests.cffi_tests.support import *
-from extra_tests.cffi_tests.support import _verify
+from extra_tests.cffi_tests.support import _verify, extra_compile_args
 import _cffi_backend
 
 lib_m = ['m']
@@ -14,18 +14,6 @@
     import distutils.ccompiler
     if distutils.ccompiler.get_default_compiler() == 'msvc':
         lib_m = ['msvcrt']
-    extra_compile_args = []      # no obvious -Werror equivalent on MSVC
-else:
-    if (sys.platform == 'darwin' and
-          [int(x) for x in os.uname()[2].split('.')] >= [11, 0, 0]):
-        # assume a standard clang or gcc
-        extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion']
-        # special things for clang
-        extra_compile_args.append('-Qunused-arguments')
-    else:
-        # assume a standard gcc
-        extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion',
-                              '-Wno-unused-parameter']
 
 class FFI(FFI):
     error = _cffi_backend.FFI.error
diff --git a/extra_tests/cffi_tests/support.py b/extra_tests/cffi_tests/support.py
--- a/extra_tests/cffi_tests/support.py
+++ b/extra_tests/cffi_tests/support.py
@@ -1,5 +1,5 @@
 # Generated by pypy/tool/import_cffi.py
-import sys
+import sys, os
 
 if sys.version_info < (3,):
     __all__ = ['u']
@@ -87,3 +87,24 @@
         if not name.startswith('_') and not hasattr(module.ffi, name):
             setattr(ffi, name, NotImplemented)
     return module.lib
+
+
+# For testing, we call gcc with "-Werror".  This is fragile because newer
+# versions of gcc are always better at producing warnings, particularly for
+# auto-generated code.  We need here to adapt and silence them as needed.
+
+if sys.platform == 'win32':
+    extra_compile_args = []      # no obvious -Werror equivalent on MSVC
+else:
+    if (sys.platform == 'darwin' and
+          [int(x) for x in os.uname()[2].split('.')] >= [11, 0, 0]):
+        # assume a standard clang or gcc
+        extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion',
+                              '-Wno-unreachable-code']
+        # special things for clang
+        extra_compile_args.append('-Qunused-arguments')
+    else:
+        # assume a standard gcc
+        extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion',
+                              '-Wno-unused-parameter',
+                              '-Wno-unreachable-code']
diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -10,7 +10,7 @@
     import os
     msg = "\n\nThe _ssl cffi module either doesn't exist or is incompatible with your machine's shared libraries.\n" + \
           "If you have a compiler installed, you can try to rebuild it by running:\n" + \
-          "cd %s\n" % os.path.abspath(os.path.dirname(os.path.dirname(__file__))) + \
+          "cd %s\n" % os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) + \
           "%s _ssl_build.py\n" % sys.executable
     raise ImportError(str(e) + msg)
 
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.13.1
+Version: 1.13.2
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -5,8 +5,8 @@
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 from .error import PkgConfigError
 
-__version__ = "1.13.1"
-__version_info__ = (1, 13, 1)
+__version__ = "1.13.2"
+__version_info__ = (1, 13, 2)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -261,12 +261,12 @@
         return (int)_cffi_to_c_wchar3216_t(o);
 }
 
-_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char32_t(int x)
+_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char32_t(unsigned int x)
 {
     if (sizeof(_cffi_wchar_t) == 4)
         return _cffi_from_c_wchar_t((_cffi_wchar_t)x);
     else
-        return _cffi_from_c_wchar3216_t(x);
+        return _cffi_from_c_wchar3216_t((int)x);
 }
 
 
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -224,7 +224,7 @@
 
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.13.1"
+                               "\ncompiled with cffi version: 1.13.2"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -159,9 +159,9 @@
 def _warn_for_non_extern_non_static_global_variable(decl):
     if not decl.storage:
         import warnings
-        warnings.warn("Declaration of global variable '%s' in cdef() should "
-                      "be marked 'extern' for consistency (or possibly "
-                      "'static' in API mode)" % (decl.name,))
+        warnings.warn("Global variable '%s' in cdef(): for consistency "
+                      "with C it should have a storage class specifier "
+                      "(usually 'extern')" % (decl.name,))
 
 def _preprocess(csource):
     # Remove comments.  NOTE: this only work because the cdef() section
diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -206,6 +206,9 @@
             from pypy.tool.pytest.objspace import gettestobjspace
             # Make cls.space and cls.runappdirect available in tests.
             spaceconfig = getattr(appclass.obj, 'spaceconfig', {})
+            config = item.config
+            if not (config.getoption('runappdirect') or config.getoption('direct_apptest')):
+                spaceconfig.setdefault('objspace.std.reinterpretasserts', True)
             appclass.obj.space = gettestobjspace(**spaceconfig)
             appclass.obj.runappdirect = option.runappdirect
 
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -1,1 +1,1 @@
-VERSION = "1.13.1"
+VERSION = "1.13.2"
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.13.1", ("This test_c.py file is for testing a version"
+assert __version__ == "1.13.2", ("This test_c.py file is for testing a version"
                                  " of cffi that differs from the one that we"
                                  " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
diff --git a/rpython/rlib/compilerinfo.py b/rpython/rlib/compilerinfo.py
--- a/rpython/rlib/compilerinfo.py
+++ b/rpython/rlib/compilerinfo.py
@@ -19,7 +19,7 @@
 
 if platform.name == 'msvc':
     # XXX hard-code the MSC version, I don't feel like computing it dynamically
-    _C_COMPILER_INFO = '"MSC v.%d 32 bit"' % (platform.version * 10 + 600)
+    _C_COMPILER_INFO = '"MSC v." Py_STR(_MSC_VER)'
 else:
     _C_COMPILER_INFO = '("GCC " __VERSION__)'
 
diff --git a/rpython/rlib/rsre/rsre_core.py b/rpython/rlib/rsre/rsre_core.py
--- a/rpython/rlib/rsre/rsre_core.py
+++ b/rpython/rlib/rsre/rsre_core.py
@@ -96,8 +96,10 @@
     def __init__(self, pattern):
         self.pattern = pattern
         # check we don't get the old value of MAXREPEAT
-        # during the untranslated tests
-        if not we_are_translated():
+        # during the untranslated tests. 
+        # On python3, MAXCODE can appear in patterns. It will be 65535
+        # when CODESIZE is 2
+        if not we_are_translated() and rsre_char.CODESIZE != 2:
             assert 65535 not in pattern
 
     def pat(self, index):
diff --git a/rpython/translator/c/src/commondefs.h b/rpython/translator/c/src/commondefs.h
--- a/rpython/translator/c/src/commondefs.h
+++ b/rpython/translator/c/src/commondefs.h
@@ -123,3 +123,7 @@
 #  define MS_WINDOWS    /* a synonym */
 #endif
 #endif
+
+/* stringify a constant */
+#define Py_XSTR(x)   #x
+#define Py_STR(x)  Py_XSTR(x)

From pypy.commits at gmail.com  Sun Nov 10 19:56:33 2019
From: pypy.commits at gmail.com (Yannick_Jadoul)
Date: Sun, 10 Nov 2019 16:56:33 -0800 (PST)
Subject: [pypy-commit] pypy backport-decode_timeval_ns-py3.7: Backport
 decode_timeval_ns from py3.7 branch
Message-ID: <5dc8b1c1.1c69fb81.7a6a8.1f3b@mx.google.com>

Author: Yannick Jadoul <yannick.jadoul at belgacom.net>
Branch: backport-decode_timeval_ns-py3.7
Changeset: r98016:37256ca2df2e
Date: 2019-11-11 01:55 +0100
http://bitbucket.org/pypy/pypy/changeset/37256ca2df2e/

Log:	Backport decode_timeval_ns from py3.7 branch

diff --git a/rpython/rlib/rtime.py b/rpython/rlib/rtime.py
--- a/rpython/rlib/rtime.py
+++ b/rpython/rlib/rtime.py
@@ -9,7 +9,7 @@
 from rpython.rtyper.tool import rffi_platform
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib.objectmodel import register_replacement_for
-from rpython.rlib.rarithmetic import intmask, UINT_MAX
+from rpython.rlib.rarithmetic import intmask, r_int64, UINT_MAX
 from rpython.rlib import rposix
 
 _WIN32 = sys.platform.startswith('win')
@@ -94,6 +94,10 @@
     return (float(rffi.getintfield(t, 'c_tv_sec')) +
             float(rffi.getintfield(t, 'c_tv_usec')) * 0.000001)
 
+def decode_timeval_ns(t):
+    return (r_int64(rffi.getintfield(t, 'c_tv_sec')) * 10**9 +
+            r_int64(rffi.getintfield(t, 'c_tv_usec')) * 10**3)
+
 
 def external(name, args, result, compilation_info=eci, **kwds):
     return rffi.llexternal(name, args, result,

From pypy.commits at gmail.com  Sun Nov 10 21:31:33 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 10 Nov 2019 18:31:33 -0800 (PST)
Subject: [pypy-commit] pypy py3.6: add missing reinterpretasserts option
Message-ID: <5dc8c805.1c69fb81.7c870.f223@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r98017:ef6a3e3097d2
Date: 2019-11-10 21:30 -0500
http://bitbucket.org/pypy/pypy/changeset/ef6a3e3097d2/

Log:	add missing reinterpretasserts option

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -218,7 +218,10 @@
         BoolOption("newshortcut",
                    "cache and shortcut calling __new__ from builtin types",
                    default=False),
-
+        BoolOption("reinterpretasserts",
+                   "Perform reinterpretation when an assert fails "
+                   "(only relevant for tests)",
+                   default=False),
      ]),
 ])
 

From pypy.commits at gmail.com  Sun Nov 10 21:47:55 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 10 Nov 2019 18:47:55 -0800 (PST)
Subject: [pypy-commit] pypy default: add " 32 bit" (with space) to version
 name on win32
Message-ID: <5dc8cbdb.1c69fb81.1aa6f.1b8e@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r98018:b3fa16549942
Date: 2019-11-10 21:46 -0500
http://bitbucket.org/pypy/pypy/changeset/b3fa16549942/

Log:	add " 32 bit" (with space) to version name on win32

diff --git a/rpython/rlib/compilerinfo.py b/rpython/rlib/compilerinfo.py
--- a/rpython/rlib/compilerinfo.py
+++ b/rpython/rlib/compilerinfo.py
@@ -18,8 +18,8 @@
 
 
 if platform.name == 'msvc':
-    # XXX hard-code the MSC version, I don't feel like computing it dynamically
-    _C_COMPILER_INFO = '"MSC v." Py_STR(_MSC_VER)'
+    # XXX hard-code the bit name
+    _C_COMPILER_INFO = '"MSC v." Py_STR(_MSC_VER) " 32 bit"'
 else:
     _C_COMPILER_INFO = '("GCC " __VERSION__)'
 

From pypy.commits at gmail.com  Mon Nov 11 07:21:51 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 11 Nov 2019 04:21:51 -0800 (PST)
Subject: [pypy-commit] pypy py3.6: merge default into py.36
Message-ID: <5dc9525f.1c69fb81.57de6.21a0@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r98019:feb30e690128
Date: 2019-11-10 21:46 -0500
http://bitbucket.org/pypy/pypy/changeset/feb30e690128/

Log:	merge default into py.36

diff --git a/rpython/rlib/compilerinfo.py b/rpython/rlib/compilerinfo.py
--- a/rpython/rlib/compilerinfo.py
+++ b/rpython/rlib/compilerinfo.py
@@ -18,8 +18,8 @@
 
 
 if platform.name == 'msvc':
-    # XXX hard-code the MSC version, I don't feel like computing it dynamically
-    _C_COMPILER_INFO = '"MSC v." Py_STR(_MSC_VER)'
+    # XXX hard-code the bit name
+    _C_COMPILER_INFO = '"MSC v." Py_STR(_MSC_VER) " 32 bit"'
 else:
     _C_COMPILER_INFO = '("GCC " __VERSION__)'
 

From pypy.commits at gmail.com  Mon Nov 11 08:07:45 2019
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 11 Nov 2019 05:07:45 -0800 (PST)
Subject: [pypy-commit] pypy py3.6: Remove reinterpretasserts again (redo
 18d56b560287)
Message-ID: <5dc95d21.1c69fb81.29693.4d5a@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6
Changeset: r98022:e5954b7c1f06
Date: 2019-11-11 13:06 +0000
http://bitbucket.org/pypy/pypy/changeset/e5954b7c1f06/

Log:	Remove reinterpretasserts again (redo 18d56b560287)

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -218,10 +218,6 @@
         BoolOption("newshortcut",
                    "cache and shortcut calling __new__ from builtin types",
                    default=False),
-        BoolOption("reinterpretasserts",
-                   "Perform reinterpretation when an assert fails "
-                   "(only relevant for tests)",
-                   default=False),
      ]),
 ])
 
diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -206,9 +206,6 @@
             from pypy.tool.pytest.objspace import gettestobjspace
             # Make cls.space and cls.runappdirect available in tests.
             spaceconfig = getattr(appclass.obj, 'spaceconfig', {})
-            config = item.config
-            if not (config.getoption('runappdirect') or config.getoption('direct_apptest')):
-                spaceconfig.setdefault('objspace.std.reinterpretasserts', True)
             appclass.obj.space = gettestobjspace(**spaceconfig)
             appclass.obj.runappdirect = option.runappdirect
 

From pypy.commits at gmail.com  Mon Nov 11 08:35:26 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 11 Nov 2019 05:35:26 -0800 (PST)
Subject: [pypy-commit] pypy wininstaller_py2: ignore '__pycache__' during
 build, hgignore build artifacts
Message-ID: <5dc9639e.1c69fb81.b223.0dc9@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: wininstaller_py2
Changeset: r98024:91bc7f57d2c8
Date: 2019-11-11 07:20 -0500
http://bitbucket.org/pypy/pypy/changeset/91bc7f57d2c8/

Log:	ignore '__pycache__' during build, hgignore build artifacts

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -74,8 +74,12 @@
 ^lib_pypy/_curses_cffi_check.c
 ^lib_pypy/_pypy_openssl.c
 ^lib_pypy/.+.o$
+^lib_pypy/.+.obj$
+^lib_pypy/.+.exp$
+^lib_pypy/.+.lib$
 ^lib_pypy/.+.so$
 ^lib_pypy/.+.pyd$
+^lib_pypy/.+.manifest$
 ^lib_pypy/Release/
 ^pypy/doc/discussion/.+\.html$
 ^include/.+\.h$
@@ -94,4 +98,5 @@
 .hypothesis/
 ^release/
 ^rpython/_cache$
-
+^PCbuild/obj$
+^PCbuild/win32$
diff --git a/pypy/tool/release/windowsinstaller/csv_to_wxs.py b/pypy/tool/release/windowsinstaller/csv_to_wxs.py
--- a/pypy/tool/release/windowsinstaller/csv_to_wxs.py
+++ b/pypy/tool/release/windowsinstaller/csv_to_wxs.py
@@ -50,6 +50,8 @@
     cache_directories = defaultdict(set)
     groups = defaultdict(list)
     for source, target, group, disk_id, condition in files:
+        if '__pycache__' in target:
+            continue
         target = PureWindowsPath(target)
         groups[group].append((source, target, disk_id, condition))
 

From pypy.commits at gmail.com  Mon Nov 11 08:35:24 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 11 Nov 2019 05:35:24 -0800 (PST)
Subject: [pypy-commit] pypy wininstaller_py2: backport wininstaller,
 PCbuild directories from windowsinstaller
Message-ID: <5dc9639c.1c69fb81.51fc1.7701@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: wininstaller_py2
Changeset: r98023:eab10f6abe32
Date: 2019-11-11 07:37 -0500
http://bitbucket.org/pypy/pypy/changeset/eab10f6abe32/

Log:	backport wininstaller, PCbuild directories from windowsinstaller

diff too long, truncating to 2000 out of 6290 lines

diff --git a/PCbuild/find_msbuild.bat b/PCbuild/find_msbuild.bat
new file mode 100644
--- /dev/null
+++ b/PCbuild/find_msbuild.bat
@@ -0,0 +1,60 @@
+ at rem
+ at rem Searches for MSBuild.exe. This is the only tool we need to initiate
+ at rem a build, so we no longer search for the full VC toolset.
+ at rem
+ at rem This file is supposed to modify the state of the caller (specifically
+ at rem the MSBUILD variable), so we do not use setlocal or echo, and avoid
+ at rem changing any other persistent state.
+ at rem
+
+ at rem No arguments provided means do full search
+ at if '%1' EQU '' goto :begin_search
+
+ at rem One argument may be the full path. Use a goto so we don't try to
+ at rem parse the next if statement - incorrect quoting in the multi-arg
+ at rem case can cause us to break immediately.
+ at if '%2' EQU '' goto :one_arg
+
+ at rem Entire command line may represent the full path if quoting failed.
+ at if exist "%*" (set MSBUILD="%*") & (set _Py_MSBuild_Source=environment) & goto :found
+ at goto :begin_search
+
+:one_arg
+ at if exist "%~1" (set MSBUILD="%~1") & (set _Py_MSBuild_Source=environment) & goto :found
+
+:begin_search
+ at set MSBUILD=
+
+ at rem If msbuild.exe is on the PATH, assume that the user wants that one.
+ at where msbuild > "%TEMP%\msbuild.loc" 2> nul && set /P MSBUILD= < "%TEMP%\msbuild.loc" & del "%TEMP%\msbuild.loc"
+ at if exist "%MSBUILD%" set MSBUILD="%MSBUILD%" & (set _Py_MSBuild_Source=PATH) & goto :found
+
+ at rem VS 2017 sets exactly one install as the "main" install, so we may find MSBuild in there.
+ at reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32 >nul 2>nul
+ at if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32') DO @(
+    @if "%%i"=="15.0" @if exist "%%k\MSBuild\15.0\Bin\msbuild.exe" @(set MSBUILD="%%k\MSBuild\15.0\Bin\msbuild.exe")
+)
+ at if exist %MSBUILD% (set _Py_MSBuild_Source=Visual Studio 2017 registry) & goto :found
+
+ at rem VS 2015 and earlier register MSBuild separately, so we can find it.
+ at reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32 >nul 2>nul
+ at if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32') DO @(
+    @if "%%i"=="MSBuildToolsPath" @if exist "%%k\msbuild.exe" @(set MSBUILD="%%k\msbuild.exe")
+)
+ at if exist %MSBUILD% (set _Py_MSBuild_Source=registry) & goto :found
+
+
+ at exit /b 1
+
+:found
+ at pushd %MSBUILD% >nul 2>nul
+ at if not ERRORLEVEL 1 @(
+  @if exist msbuild.exe @(set MSBUILD="%CD%\msbuild.exe") else @(set MSBUILD=)
+  @popd
+)
+
+ at if defined MSBUILD @echo Using %MSBUILD% (found in the %_Py_MSBuild_Source%)
+ at if not defined MSBUILD @echo Failed to find MSBuild
+ at set _Py_MSBuild_Source=
+ at if not defined MSBUILD @exit /b 1
+ at exit /b 0
diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props
new file mode 100644
--- /dev/null
+++ b/PCbuild/pyproject.props
@@ -0,0 +1,197 @@
+&#65279;<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003" TreatAsLocalProperty="Py_IntDir">
+  <PropertyGroup Label="Globals">
+    <_ProjectFileVersion>10.0.30319.1</_ProjectFileVersion>
+    <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">10.0</VisualStudioVersion>
+    <OutDir>$(BuildPath)</OutDir>
+    <OutDir Condition="!HasTrailingSlash($(OutDir))">$(OutDir)\</OutDir>
+    <Py_IntDir Condition="'$(Py_IntDir)' == ''">$(MSBuildThisFileDirectory)obj\</Py_IntDir>
+    <IntDir>$(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\$(ProjectName)\</IntDir>
+    <IntDir>$(IntDir.Replace(`\\`, `\`))</IntDir>
+    <TargetName Condition="'$(TargetName)' == ''">$(ProjectName)</TargetName>
+    <TargetName>$(TargetName)$(PyDebugExt)</TargetName>
+    <GenerateManifest>false</GenerateManifest>
+    <EmbedManifest>false</EmbedManifest>
+    <SupportPGO Condition="'$(SupportPGO)' == ''">true</SupportPGO>
+    <SupportSigning Condition="'$(SupportSigning)' == ''">true</SupportSigning>
+    <SupportSigning Condition="'$(Configuration)' == 'Debug'">false</SupportSigning>
+    <SupportSigning Condition="'$(ConfigurationType)' == 'StaticLibrary'">false</SupportSigning>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <_DebugPreprocessorDefinition>NDEBUG;</_DebugPreprocessorDefinition>
+    <_DebugPreprocessorDefinition Condition="$(Configuration) == 'Debug'">_DEBUG;</_DebugPreprocessorDefinition>
+    <_PlatformPreprocessorDefinition>_WIN32;</_PlatformPreprocessorDefinition>
+    <_PlatformPreprocessorDefinition Condition="$(Platform) == 'x64'">_WIN64;_M_X64;</_PlatformPreprocessorDefinition>
+    <_PydPreprocessorDefinition Condition="$(TargetExt) == '.pyd'">Py_BUILD_CORE_MODULE;</_PydPreprocessorDefinition>
+  </PropertyGroup>
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <AdditionalIncludeDirectories>$(PySourcePath)Include;$(PySourcePath)Include\internal;$(PySourcePath)PC;$(IntDir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;$(_PlatformPreprocessorDefinition)$(_DebugPreprocessorDefinition)$(_PydPreprocessorDefinition)%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      
+      <Optimization>MaxSpeed</Optimization>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <StringPooling>true</StringPooling>
+      <ExceptionHandling></ExceptionHandling>
+      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <WarningLevel>Level3</WarningLevel>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+      <CompileAs>Default</CompileAs>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+      <WholeProgramOptimization>true</WholeProgramOptimization>
+      <EnableEnhancedInstructionSet Condition="'$(Platform)'=='Win32'">NoExtensions</EnableEnhancedInstructionSet>
+      <InlineFunctionExpansion Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">OnlyExplicitInline</InlineFunctionExpansion>
+      <InlineFunctionExpansion Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">OnlyExplicitInline</InlineFunctionExpansion>
+    </ClCompile>
+    <ClCompile Condition="$(Configuration) == 'Debug'">
+      <Optimization>Disabled</Optimization>
+      <WholeProgramOptimization>false</WholeProgramOptimization>
+      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
+    </ClCompile>
+    <ClCompile Condition="$(ICCBuild) == 'true'">
+      <FloatingPointModel>Strict</FloatingPointModel>
+    </ClCompile>
+    <Link>
+      <AdditionalLibraryDirectories>$(OutDir);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <ProgramDatabaseFile>$(OutDir)$(TargetName).pdb</ProgramDatabaseFile>
+      <SubSystem>Windows</SubSystem>
+      <RandomizedBaseAddress>true</RandomizedBaseAddress>
+      <DataExecutionPrevention>true</DataExecutionPrevention>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+      <IgnoreSpecificDefaultLibraries>LIBC;%(IgnoreSpecificDefaultLibraries)</IgnoreSpecificDefaultLibraries>
+      <TargetMachine>MachineX86</TargetMachine>
+      <TargetMachine Condition="'$(Platform)' == 'x64'">MachineX64</TargetMachine>
+      <ProfileGuidedDatabase Condition="$(SupportPGO)">$(OutDir)$(TargetName).pgd</ProfileGuidedDatabase>
+      <LinkTimeCodeGeneration Condition="$(Configuration) == 'Release'">UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
+      <LinkTimeCodeGeneration Condition="$(SupportPGO) and $(Configuration) == 'PGInstrument'">PGInstrument</LinkTimeCodeGeneration>
+      <LinkTimeCodeGeneration Condition="$(SupportPGO) and $(Configuration) == 'PGUpdate'">PGUpdate</LinkTimeCodeGeneration>
+    </Link>
+    <Lib>
+      <LinkTimeCodeGeneration Condition="$(Configuration) == 'Release'">true</LinkTimeCodeGeneration>
+      <LinkTimeCodeGeneration Condition="$(SupportPGO) and $(Configuration) == 'PGInstrument'">true</LinkTimeCodeGeneration>
+      <LinkTimeCodeGeneration Condition="$(SupportPGO) and $(Configuration) == 'PGUpdate'">true</LinkTimeCodeGeneration>
+    </Lib>
+    <ResourceCompile>
+      <AdditionalIncludeDirectories>$(PySourcePath)PC;$(PySourcePath)Include;$(IntDir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>$(_DebugPreprocessorDefinition)%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <Culture>0x0409</Culture>
+    </ResourceCompile>
+    <Midl>
+      <PreprocessorDefinitions>$(_DebugPreprocessorDefinition)%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <MkTypLibCompatible>true</MkTypLibCompatible>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+      <TargetEnvironment>Win32</TargetEnvironment>
+      <TargetEnvironment Condition="'$(Platform)' == 'x64'">X64</TargetEnvironment>
+      <OutputDirectory>$(IntDir)</OutputDirectory>
+      <InterfaceIdentifierFileName>$(MSBuildProjectName)_i.c</InterfaceIdentifierFileName>
+      <ProxyFileName>$(MSBuildProjectName)_p.c</ProxyFileName>
+    </Midl>
+  </ItemDefinitionGroup>
+
+  <Target Name="GeneratePythonNtRcH"
+          BeforeTargets="ClCompile"
+          Inputs="$(PySourcePath)Include\patchlevel.h"
+          Outputs="$(IntDir)pythonnt_rc.h">
+    <WriteLinesToFile File="$(IntDir)pythonnt_rc.h" Overwrite="true" Encoding="ascii"
+                      Lines='/* This file created by pyproject.props /t:GeneratePythonNtRcH */
+#define FIELD3 $(Field3Value)
+#define MS_DLL_ID "$(SysWinVer)"
+#define PYTHON_DLL_NAME "$(TargetName)$(TargetExt)"
+' />
+    <ItemGroup>
+        <FileWrites Include="$(IntDir)pythonnt_rc.h" />
+    </ItemGroup>
+  </Target>
+
+  <UsingTask TaskName="KillPython" TaskFactory="CodeTaskFactory" AssemblyFile="$(MSBuildToolsPath)\Microsoft.Build.Tasks.v4.0.dll">
+    <ParameterGroup>
+      <FileName Required="true" />
+    </ParameterGroup>
+    <Task>
+      <Using Namespace="System.Diagnostics"/>
+      <Using Namespace="System.IO"/>
+      <Using Namespace="System.Runtime.InteropServices"/>
+      <Using Namespace="System.Text"/>
+      <Code Type="Method" Language="cs">
+<![CDATA[
+[DllImport("kernel32.dll", SetLastError=true, CharSet=CharSet.Unicode)]
+public static extern bool QueryFullProcessImageName([In]IntPtr hProcess, [In]int dwFlags,
+                                                    [Out]StringBuilder lpExeName, ref int lpdwSize);
+public override bool Execute() {
+    string fullPath = Path.GetFullPath(FileName);
+    Log.LogMessage("Looking for " + fullPath, MessageImportance.Normal);
+    foreach (Process p in Process.GetProcesses()) {
+        try {
+            int pathLength = 32768;
+            StringBuilder pathBuilder = new StringBuilder(pathLength);
+            if (QueryFullProcessImageName(p.Handle, 0, pathBuilder, ref pathLength)) {
+                string exeName = Path.GetFullPath(pathBuilder.ToString());
+                Log.LogMessage("Found running process: " + exeName, MessageImportance.Low);
+                if (fullPath.Equals(exeName, StringComparison.OrdinalIgnoreCase)) {
+                    Log.LogMessage("Terminating " + exeName, MessageImportance.High);
+                    p.Kill();
+                }
+            }
+        } catch {
+        }
+    }
+    return true;
+}
+]]>
+      </Code>
+    </Task>
+  </UsingTask>
+  
+  <Target Name="KillPython" BeforeTargets="PrepareForBuild" Condition="'$(KillPython)' == 'true'">
+    <Message Text="Killing any running python$(PyDebugExt)$(PyTestExt).exe instances..." Importance="high" />
+    <KillPython FileName="$(OutDir)python$(PyDebugExt)$(PyTestExt).exe" />
+  </Target>
+  
+  <!--
+  A default target to handle msbuild pcbuild.proj /t:CleanAll.
+  
+  Some externals projects don't respond to /t:Clean, so we invoke
+  CleanAll on them when we really want to clean up.
+  -->
+  <Target Name="CleanAll" DependsOnTargets="Clean">
+    <MSBuild Projects="@(ProjectReference->'%(FullPath)')"
+             Properties="Configuration=$(Configuration);Platform=$(Platform)"
+             BuildInParallel="true"
+             StopOnFirstFailure="false"
+             Condition="Exists(%(FullPath))"
+             Targets="CleanAll" />
+  </Target>
+
+  <Target Name="CopyPGCFiles" BeforeTargets="PrepareForBuild" Condition="$(Configuration) == 'PGUpdate'">
+    <ItemGroup>
+      <_PGCFiles Include="$(OutDir)instrumented\$(TargetName)!*.pgc" />
+      <_PGDFile Include="$(OutDir)instrumented\$(TargetName).pgd" />
+      <_CopyFiles Include="@(_PGCFiles);@(_PGDFile)" Condition="Exists(%(FullPath))" />
+    </ItemGroup>
+    <Delete Files="@(_CopyFiles->'$(OutDir)%(Filename)%(Extension)')" />
+    <Error Text="PGO run did not succeed (no $(TargetName)!*.pgc files) and there is no data to merge"
+           Condition="$(RequirePGCFiles) == 'true' and @(_PGCFiles) == ''" />
+    <Copy SourceFiles="@(_CopyFiles)"
+          DestinationFolder="$(OutDir)"
+          UseHardLinksIfPossible="true"
+          OverwriteReadOnlyFiles="true" />
+  </Target>
+
+  <PropertyGroup>
+    <SdkBinPath Condition="'$(SdkBinPath)' == '' or !Exists($(SdkBinPath))">$(registry:HKEY_LOCAL_MACHINE\Software\Microsoft\Windows Kits\Installed Roots at KitsRoot10)\bin\$(DefaultWindowsSDKVersion)\x86</SdkBinPath>
+    <SdkBinPath Condition="!Exists($(SdkBinPath))">$(registry:HKEY_LOCAL_MACHINE\Software\Microsoft\Windows Kits\Installed Roots at KitsRoot10)\bin\x86</SdkBinPath>
+    <SdkBinPath Condition="!Exists($(SdkBinPath))">$(registry:HKEY_LOCAL_MACHINE\Software\Microsoft\Windows Kits\Installed Roots at KitsRoot81)\bin\x86</SdkBinPath>
+    <SdkBinPath Condition="!Exists($(SdkBinPath))">$(registry:HKEY_LOCAL_MACHINE\Software\Microsoft\Windows Kits\Installed Roots at KitsRoot)\bin\x86</SdkBinPath>
+    <SdkBinPath Condition="!Exists($(SdkBinPath))">$(registry:HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Microsoft SDKs\Windows\v7.1A at InstallationFolder)\Bin\</SdkBinPath>
+    <_SignCommand Condition="Exists($(SdkBinPath)) and '$(SigningCertificate)' != '' and $(SupportSigning)">"$(SdkBinPath)\signtool.exe" sign /q /a /n "$(SigningCertificate)" /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d "Python $(PythonVersion)"</_SignCommand>
+    <_MakeCatCommand Condition="Exists($(SdkBinPath))">"$(SdkBinPath)\makecat.exe"</_MakeCatCommand>
+  </PropertyGroup>
+  
+  <Target Name="_SignBuild" AfterTargets="AfterBuild" Condition="'$(SigningCertificate)' != '' and $(SupportSigning)">
+    <Error Text="Unable to locate signtool.exe. Set /p:SignToolPath and rebuild" Condition="'$(_SignCommand)' == ''" />
+    <Exec Command='$(_SignCommand) "$(TargetPath)" || $(_SignCommand) "$(TargetPath)" || $(_SignCommand) "$(TargetPath)"' ContinueOnError="false" />
+  </Target>
+</Project>
diff --git a/PCbuild/python.props b/PCbuild/python.props
new file mode 100644
--- /dev/null
+++ b/PCbuild/python.props
@@ -0,0 +1,193 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <Platform Condition="'$(Platform)' == ''">Win32</Platform>
+    <Configuration Condition="'$(Configuration)' == ''">Release</Configuration>
+    <!--
+    Use the latest available version of Visual Studio to build. To override
+    this and build with an earlier version, pass "/p:PlatformToolset=v100"
+    (for example) when building.
+
+    We set BasePlatformToolset for ICC's benefit, it's otherwise ignored.
+    -->
+    <BasePlatformToolset Condition="'$(BasePlatformToolset)' == '' and ('$(MSBuildToolsVersion)' == '15.0' or '$(VisualStudioVersion)' == '15.0')">v141</BasePlatformToolset>
+    <BasePlatformToolset Condition="'$(BasePlatformToolset)' == '' and '$(VCTargetsPath14)' != ''">v140</BasePlatformToolset>
+    <BasePlatformToolset Condition="'$(BasePlatformToolset)' == '' and '$(VCTargetsPath12)' != ''">v120</BasePlatformToolset>
+    <BasePlatformToolset Condition="'$(BasePlatformToolset)' == '' and '$(VCTargetsPath11)' != ''">v110</BasePlatformToolset>
+    <BasePlatformToolset Condition="'$(BasePlatformToolset)' == '' and '$(VCTargetsPath10)' != ''">v100</BasePlatformToolset>
+
+    <PlatformToolset Condition="'$(PlatformToolset)' == ''">$(BasePlatformToolset)</PlatformToolset>
+    <ICCBuild>false</ICCBuild>
+    <ICCBuild Condition="$(PlatformToolset.StartsWith(`Intel C++ Compiler`))">true</ICCBuild>
+
+    <!--
+    Convincing MSVC/MSBuild to prefer our platform names is too difficult,
+    so we define our own constant ArchName and use wherever we need it.
+    -->
+    <ArchName Condition="'$(ArchName)' == '' and $(Platform) == 'x64'">amd64</ArchName>
+    <ArchName Condition="'$(ArchName)' == ''">win32</ArchName>
+    
+    <!-- Root directory of the repository -->
+    <PySourcePath Condition="'$(PySourcePath)' == ''">$([System.IO.Path]::GetFullPath($(MSBuildThisFileDirectory)\..\))</PySourcePath>
+    <PySourcePath Condition="!HasTrailingSlash($(PySourcePath))">$(PySourcePath)\</PySourcePath>
+    
+	<!-- Directory containing the translated python interpretter -->
+	<InterpreterPath Condition="'$(InterpreterPath)' == ''">$(PySourcePath)pypy\goal\</InterpreterPath>
+    
+	<!-- Directory where msi build outputs are put -->
+	<BuildPath32 Condition="'$(Py_OutDir)' == ''">$(PySourcePath)PCbuild\win32\</BuildPath32>
+    <BuildPath32 Condition="'$(Py_OutDir)' != ''">$(Py_OutDir)\win32\</BuildPath32>
+    <BuildPath Condition="'$(BuildPath)' == ''">$(BuildPath32)</BuildPath>
+    <BuildPath Condition="!HasTrailingSlash($(BuildPath))">$(BuildPath)\</BuildPath>
+    <BuildPath Condition="$(Configuration) == 'PGInstrument'">$(BuildPath)instrumented\</BuildPath>
+        
+    <!-- Directories of external projects. tcltk is handled in tcltk.props -->
+    <ExternalsDir>$(EXTERNALS_DIR)</ExternalsDir>
+    <ExternalsDir Condition="$(ExternalsDir) == ''">$([System.IO.Path]::GetFullPath(`$(PySourcePath)externals`))</ExternalsDir>
+    <ExternalsDir Condition="!HasTrailingSlash($(ExternalsDir))">$(ExternalsDir)\</ExternalsDir>
+    <sqlite3Dir>$(ExternalsDir)sqlite-3.21.0.0\</sqlite3Dir>
+    <bz2Dir>$(ExternalsDir)bzip2-1.0.6\</bz2Dir>
+    <lzmaDir>$(ExternalsDir)xz-5.2.2\</lzmaDir>
+    <opensslDir>$(ExternalsDir)openssl-1.1.0i\</opensslDir>
+    <opensslOutDir>$(ExternalsDir)openssl-bin-1.1.0i\$(ArchName)\</opensslOutDir>
+    <opensslIncludeDir>$(opensslOutDir)include</opensslIncludeDir>
+    <nasmDir>$(ExternalsDir)\nasm-2.11.06\</nasmDir>
+    <zlibDir>$(ExternalsDir)\zlib-1.2.11\</zlibDir>
+    
+    <!-- Suffix for all binaries when building for debug -->
+    <PyDebugExt Condition="'$(PyDebugExt)' == '' and $(Configuration) == 'Debug'">_d</PyDebugExt>
+    
+    <!-- Suffix for versions/keys when building with test markers -->
+    <PyTestExt Condition="$(UseTestMarker) == 'true'">-test</PyTestExt>
+    
+    <!-- Suffix for versions/keys when building for particular platforms -->
+    <PyArchExt Condition="'$(ArchName)' == 'win32'">-32</PyArchExt>
+    
+    <!-- Full path of the resulting python.exe binary -->
+    <PythonExe Condition="'$(PythonExe)' == ''">$(InterpreterPath)pypy3-c$(PyDebugExt).exe</PythonExe>
+  </PropertyGroup>
+  
+  <PropertyGroup Condition="$(DefaultWindowsSDKVersion) == ''">
+    <!--
+    Attempt to select the latest installed WinSDK. If we don't find any, then we will
+    let the MSBuild targets determine which one it wants to use (typically the earliest
+    possible version). Since we limit WINVER to Windows 7 anyway, it doesn't really
+    matter which WinSDK version we use.
+    -->
+    <_RegistryVersion>$(Registry:HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Microsoft SDKs\Windows\v10.0 at ProductVersion)</_RegistryVersion>
+    <_RegistryVersion Condition="$(_RegistryVersion) == ''">$(Registry:HKEY_LOCAL_MACHINE\WOW6432Node\SOFTWARE\Microsoft\Microsoft SDKs\Windows\v10.0 at ProductVersion)</_RegistryVersion>
+    <DefaultWindowsSDKVersion>10.0.17134.0</DefaultWindowsSDKVersion>
+    <DefaultWindowsSDKVersion Condition="$(_RegistryVersion) == '10.0.16299'">10.0.16299.0</DefaultWindowsSDKVersion>
+    <DefaultWindowsSDKVersion Condition="$(_RegistryVersion) == '10.0.15063'">10.0.15063.0</DefaultWindowsSDKVersion>
+    <DefaultWindowsSDKVersion Condition="$(_RegistryVersion) == '10.0.14393'">10.0.14393.0</DefaultWindowsSDKVersion>
+    <DefaultWindowsSDKVersion Condition="$(_RegistryVersion) == '10.0.10586'">10.0.10586.0</DefaultWindowsSDKVersion>
+    <DefaultWindowsSDKVersion Condition="$(_RegistryVersion) == '10.0.10240'">10.0.10240.0</DefaultWindowsSDKVersion>
+  </PropertyGroup>
+  
+  <PropertyGroup Condition="'$(OverrideVersion)' == ''">
+    <!--
+    Read version information from Include\patchlevel.h. The following properties are set:
+    
+        MajorVersionNumber  -   the '3' in '3.5.2a1'
+        MinorVersionNumber  -   the '5' in '3.5.2a1'
+        MicroVersionNumber  -   the '2' in '3.5.2a1'
+        ReleaseSerial       -   the '1' in '3.5.2a1'
+        ReleaseLevelName    -   the 'a1' in '3.5.2a1'
+        PythonVersionNumber -   '3.5.2' for '3.5.2a1'
+        PythonVersion       -   '3.5.2a1'
+        PythonVersionHex    -   0x030502a1 for '3.5.2a1'
+        ReleaseLevelNumber  -   10 for alpha, 11 for beta, 12 for RC (gamma), and 15 for final
+        Field3Value         -   2101 for '3.5.2a1' (== 1000*2 + 10*10 ('a') + 1)
+    -->
+    <_PatchLevelContent>$([System.IO.File]::ReadAllText(`$(PySourcePath)Include\patchlevel.h`))</_PatchLevelContent>
+    <MajorVersionNumber>$([System.Text.RegularExpressions.Regex]::Match($(_PatchLevelContent), `define\s+PY_MAJOR_VERSION\s+(\d+)`).Groups[1].Value)</MajorVersionNumber>
+    <MinorVersionNumber>$([System.Text.RegularExpressions.Regex]::Match($(_PatchLevelContent), `define\s+PY_MINOR_VERSION\s+(\d+)`).Groups[1].Value)</MinorVersionNumber>
+    <MicroVersionNumber>$([System.Text.RegularExpressions.Regex]::Match($(_PatchLevelContent), `define\s+PY_MICRO_VERSION\s+(\d+)`).Groups[1].Value)</MicroVersionNumber>
+    <_ReleaseLevel>$([System.Text.RegularExpressions.Regex]::Match($(_PatchLevelContent), `define\s+PY_RELEASE_LEVEL\s+PY_RELEASE_LEVEL_(\w+)`).Groups[1].Value)</_ReleaseLevel>
+    <ReleaseSerial>$([System.Text.RegularExpressions.Regex]::Match($(_PatchLevelContent), `define\s+PY_RELEASE_SERIAL\s+(\d+)`).Groups[1].Value)</ReleaseSerial>
+    <ReleaseLevelNumber>15</ReleaseLevelNumber>
+    <ReleaseLevelNumber Condition="$(_ReleaseLevel) == 'ALPHA'">10</ReleaseLevelNumber>
+    <ReleaseLevelNumber Condition="$(_ReleaseLevel) == 'BETA'">11</ReleaseLevelNumber>
+    <ReleaseLevelNumber Condition="$(_ReleaseLevel) == 'GAMMA'">12</ReleaseLevelNumber>
+    <ReleaseLevelName Condition="$(_ReleaseLevel) == 'ALPHA'">a$(ReleaseSerial)</ReleaseLevelName>
+    <ReleaseLevelName Condition="$(_ReleaseLevel) == 'BETA'">b$(ReleaseSerial)</ReleaseLevelName>
+    <ReleaseLevelName Condition="$(_ReleaseLevel) == 'GAMMA'">rc$(ReleaseSerial)</ReleaseLevelName>
+  </PropertyGroup>
+  
+  <PropertyGroup Condition="'$(OverrideVersion)' != ''">
+    <!--
+    Override the version number when building by specifying OverrideVersion.
+    For example:
+    
+        PCbuild\build.bat "/p:OverrideVersion=3.5.2a1"
+    
+    Use the -V option to check your version is valid:
+    
+        PCbuild\build.bat -V "/p:OverrideVersion=3.5.2a1"
+          PythonVersionNumber: 3.5.2
+          PythonVersion:       3.5.2a1
+          PythonVersionHex:    0x030502A1
+          Field3Value:         2101
+    
+    Note that this only affects the version numbers embedded in resources and
+    installers, but not sys.version.
+    -->
+    <MajorVersionNumber>$([System.Text.RegularExpressions.Regex]::Match($(OverrideVersion), `(\d+)\.(\d+)\.(\d+)((a|b|rc)(\d))?`).Groups[1].Value)</MajorVersionNumber>
+    <MinorVersionNumber>$([System.Text.RegularExpressions.Regex]::Match($(OverrideVersion), `(\d+)\.(\d+)\.(\d+)((a|b|rc)(\d))?`).Groups[2].Value)</MinorVersionNumber>
+    <MicroVersionNumber>$([System.Text.RegularExpressions.Regex]::Match($(OverrideVersion), `(\d+)\.(\d+)\.(\d+)((a|b|rc)(\d))?`).Groups[3].Value)</MicroVersionNumber>
+    <ReleaseLevelName>$([System.Text.RegularExpressions.Regex]::Match($(OverrideVersion), `(\d+)\.(\d+)\.(\d+)((a|b|rc)(\d))?`).Groups[4].Value)</ReleaseLevelName>
+    <_ReleaseLevel>$([System.Text.RegularExpressions.Regex]::Match($(OverrideVersion), `(\d+)\.(\d+)\.(\d+)((a|b|rc)(\d))?`).Groups[5].Value)</_ReleaseLevel>
+    <ReleaseSerial>$([System.Text.RegularExpressions.Regex]::Match($(OverrideVersion), `(\d+)\.(\d+)\.(\d+)((a|b|rc)(\d))?`).Groups[6].Value)</ReleaseSerial>
+    <ReleaseSerial Condition="'$(ReleaseSerial)' == ''">0</ReleaseSerial>
+    <ReleaseLevelNumber>15</ReleaseLevelNumber>
+    <ReleaseLevelNumber Condition="$(_ReleaseLevel) == 'a'">10</ReleaseLevelNumber>
+    <ReleaseLevelNumber Condition="$(_ReleaseLevel) == 'b'">11</ReleaseLevelNumber>
+    <ReleaseLevelNumber Condition="$(_ReleaseLevel) == 'rc'">12</ReleaseLevelNumber>
+  </PropertyGroup>
+  
+  <PropertyGroup>
+    <PythonVersionNumber>$(MajorVersionNumber).$(MinorVersionNumber).$(MicroVersionNumber)</PythonVersionNumber>
+    <PythonVersion>$(MajorVersionNumber).$(MinorVersionNumber).$(MicroVersionNumber)$(ReleaseLevelName)</PythonVersion>
+    <PythonVersionHex>$([msbuild]::BitwiseOr(
+        $([msbuild]::Multiply($(MajorVersionNumber), 16777216)),
+        $([msbuild]::BitwiseOr(
+            $([msbuild]::Multiply($(MinorVersionNumber), 65536)),
+            $([msbuild]::BitwiseOr(
+                $([msbuild]::Multiply($(MicroVersionNumber), 256)),
+                $([msbuild]::BitwiseOr(
+                    $([msbuild]::Multiply($(ReleaseLevelNumber), 16)),
+                    $(ReleaseSerial)
+                ))
+            ))
+        ))
+    ))</PythonVersionHex>
+    <Field3Value>$([msbuild]::Add(
+        $(ReleaseSerial),
+        $([msbuild]::Add(
+            $([msbuild]::Multiply($(ReleaseLevelNumber), 10)),
+            $([msbuild]::Multiply($(MicroVersionNumber), 1000))
+        ))
+    ))</Field3Value>
+    <Field3Value Condition="$(UseTestMarker) == 'true'">$([msbuild]::Add($(Field3Value), 9000))</Field3Value>
+    
+    <!-- The name of the resulting pythonXY.dll (without the extension) -->
+    <PyDllName>python$(MajorVersionNumber)$(MinorVersionNumber)$(PyDebugExt)</PyDllName>
+
+    <!-- The version and platform tag to include in .pyd filenames -->
+    <PydTag Condition="$(ArchName) == 'win32'">.cp$(MajorVersionNumber)$(MinorVersionNumber)-win32</PydTag>
+    <PydTag Condition="$(ArchName) == 'amd64'">.cp$(MajorVersionNumber)$(MinorVersionNumber)-win_amd64</PydTag>
+    
+    <!-- The version number for sys.winver -->
+    <SysWinVer>$(MajorVersionNumber).$(MinorVersionNumber)$(PyArchExt)$(PyTestExt)</SysWinVer>
+  </PropertyGroup>
+  
+  <!-- Displays the calculated version info -->
+  <Target Name="ShowVersionInfo">
+    <Message Importance="high" Text="PythonVersionNumber: $(PythonVersionNumber)" />
+    <Message Importance="high" Text="PythonVersion:       $(PythonVersion)" />
+    <Message Importance="high" Text="PythonVersionHex:    0x$([System.UInt32]::Parse($(PythonVersionHex)).ToString(`X08`))" />
+    <Message Importance="high" Text="Field3Value:         $(Field3Value)" />
+    <Message Importance="high" Text="SysWinVer:           $(SysWinVer)" />
+    <Message Importance="high" Text="PyDllName:           $(PyDllName)" />
+  </Target>
+</Project>
diff --git a/PCbuild/tcltk.props b/PCbuild/tcltk.props
new file mode 100644
--- /dev/null
+++ b/PCbuild/tcltk.props
@@ -0,0 +1,45 @@
+&#65279;<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="pyproject.props" />
+  <PropertyGroup>
+    <TclMajorVersion>8</TclMajorVersion>
+    <TclMinorVersion>6</TclMinorVersion>
+    <TclPatchLevel>8</TclPatchLevel>
+    <TclRevision>0</TclRevision>
+    <TkMajorVersion>$(TclMajorVersion)</TkMajorVersion>
+    <TkMinorVersion>$(TclMinorVersion)</TkMinorVersion>
+    <TkPatchLevel>$(TclPatchLevel)</TkPatchLevel>
+    <TkRevision>$(TclRevision)</TkRevision>
+    <TixMajorVersion>8</TixMajorVersion>
+    <TixMinorVersion>4</TixMinorVersion>
+    <TixPatchLevel>3</TixPatchLevel>
+    <TixRevision>6</TixRevision>
+    <tclDir>$(ExternalsDir)tcl-core-$(TclMajorVersion).$(TclMinorVersion).$(TclPatchLevel).$(TclRevision)\</tclDir>
+    <tkDir>$(ExternalsDir)tk-$(TkMajorVersion).$(TkMinorVersion).$(TkPatchLevel).$(TkRevision)\</tkDir>
+    <tixDir>$(ExternalsDir)tix-$(TixMajorVersion).$(TixMinorVersion).$(TixPatchLevel).$(TixRevision)\</tixDir>
+    <tcltkDir>$(ExternalsDir)tcltk-$(TclMajorVersion).$(TclMinorVersion).$(TclPatchLevel).$(TclRevision)\$(ArchName)\</tcltkDir>
+    <!--<TclDebugExt Condition="'$(Configuration)' == 'Debug'">g</TclDebugExt>-->
+    <tclDLLName>tcl$(TclMajorVersion)$(TclMinorVersion)t$(TclDebugExt).dll</tclDLLName>
+    <tclLibName>tcl$(TclMajorVersion)$(TclMinorVersion)t$(TclDebugExt).lib</tclLibName>
+    <tclShExeName>tclsh$(TclMajorVersion)$(TclMinorVersion)t$(TclDebugExt).exe</tclShExeName>
+    <tkDLLName>tk$(TkMajorVersion)$(TkMinorVersion)t$(TclDebugExt).dll</tkDLLName>
+    <tkLibName>tk$(TkMajorVersion)$(TkMinorVersion)t$(TclDebugExt).lib</tkLibName>
+    <tixDLLName>tix$(TixMajorVersion)$(TixMinorVersion)$(TclDebugExt).dll</tixDLLName>
+    <tixDLLPath>$(tcltkDir)lib\tix$(TixMajorVersion).$(TixMinorVersion).$(TixPatchLevel)\$(tixDLLName)</tixDLLPath>
+    <tcltkLib>$(tcltkDir)lib\tcl$(TclMajorVersion)$(TclMinorVersion)t$(TclDebugExt).lib;$(tcltkDir)lib\tk$(TkMajorVersion)$(TkMinorVersion)t$(TclDebugExt).lib</tcltkLib>
+    <TclMachine>IX86</TclMachine>
+    <TclMachine Condition="'$(Platform)' == 'x64'">AMD64</TclMachine>
+    <TclVersions>TCL_MAJOR_VERSION=$(TclMajorVersion) TCL_MINOR_VERSION=$(TclMinorVersion) TCL_PATCH_LEVEL=$(TclPatchLevel)</TclVersions>
+    <TclShortVersions>TCL_MAJOR=$(TclMajorVersion) TCL_MINOR=$(TclMinorVersion) TCL_PATCH=$(TclPatchLevel)</TclShortVersions>
+    <TkVersions>TK_MAJOR_VERSION=$(TkMajorVersion) TK_MINOR_VERSION=$(TkMinorVersion) TK_PATCH_LEVEL=$(TkPatchLevel)</TkVersions>
+
+    <BuildDirTop>Release</BuildDirTop>
+    <BuildDirTop Condition="$(Configuration) == 'Debug'">Debug</BuildDirTop>
+    <BuildDirTop Condition="$(TclMachine) != 'IX86'">$(BuildDirTop)_$(TclMachine)</BuildDirTop>
+    <BuildDirTop Condition="$(PlatformToolset) == 'v141'">$(BuildDirTop)_VC13</BuildDirTop>
+    <BuildDirTop Condition="$(PlatformToolset) == 'v140'">$(BuildDirTop)_VC13</BuildDirTop>
+    <BuildDirTop Condition="$(PlatformToolset) == 'v120'">$(BuildDirTop)_VC12</BuildDirTop>
+    <BuildDirTop Condition="$(PlatformToolset) == 'v110'">$(BuildDirTop)_VC11</BuildDirTop>
+    <BuildDirTop Condition="$(PlatformToolset) == 'v100'">$(BuildDirTop)_VC10</BuildDirTop>
+  </PropertyGroup>
+</Project>
diff --git a/PCbuild/win32/en-us/exe.msi b/PCbuild/win32/en-us/exe.msi
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..a444b52839ec6e9ad8eadde9653e7dd370910671
GIT binary patch

[cut]

diff --git a/pypy/tool/release/windowsinstaller/README b/pypy/tool/release/windowsinstaller/README
new file mode 100644
--- /dev/null
+++ b/pypy/tool/release/windowsinstaller/README
@@ -0,0 +1,15 @@
+Prerequisites:
+    Visual Studio 2017 or at least the compilers and desktop cevelopment
+with c++
+        .Net 3.5.1
+    Wix 3.11.1 http://wixtoolset.org/releases/
+    Wix VSExtension for VS 2017
+https://marketplace.visualstudio.com/items?itemName=RobMensching.WixToolsetVisualStudio2017Extension
+    Windows 10 SDK (10.0.17134.0)
+    
+Steps:
+    1. Translate the interpretter from the \pypy\goal folder.
+    2. Remove all __pycache__ directories (maybe do this in the script?)
+    3. Run buildrelease.bat from the visual studio command line.
+
+Now you should have .\PCbuild\win32\en-us\pypy-3.5.3.6609.exe
diff --git a/pypy/tool/release/windowsinstaller/buildrelease.bat b/pypy/tool/release/windowsinstaller/buildrelease.bat
new file mode 100644
--- /dev/null
+++ b/pypy/tool/release/windowsinstaller/buildrelease.bat
@@ -0,0 +1,44 @@
+ at setlocal
+ at echo off
+
+
+set D=%~dp0
+set PCBUILD=%D%..\..\..\..\PCbuild\
+if "%Py_OutDir%"=="" set Py_OutDir=%PCBUILD%
+set EXTERNALS=%D%..\..\externals\windows-installer\
+
+set BUILDX86=
+set BUILDX64=
+set TARGET=Rebuild
+set TESTTARGETDIR=
+set PGO=-m test -q --pgo
+set BUILDNUGET=1
+set BUILDZIP=1
+
+
+:CheckOpts
+if "%1" EQU "-h" goto Help
+if "%1" EQU "-o" (set OUTDIR=%~2) && shift && shift && goto CheckOpts
+if "%1" EQU "--out" (set OUTDIR=%~2) && shift && shift && goto CheckOpts
+
+
+if "%1" NEQ "" echo Invalid option: "%1" && exit /B 1
+
+if not defined BUILDX86 if not defined BUILDX64 (set BUILDX86=1) && (set BUILDX64=1)
+
+call "%PCBUILD%find_msbuild.bat" %MSBUILD%
+if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2)
+
+
+%MSBUILD% "%D%bundle\releaselocal.wixproj" /t:Rebuild /p:RebuildAll=true
+if errorlevel 1 exit /B
+
+exit /B 0
+
+:Help
+echo buildrelease.bat [--out DIR]
+echo                  [-h]
+echo.
+echo    --out (-o)          Specify an additional output directory for installers
+echo    -h                  Display this help information
+echo.
diff --git a/pypy/tool/release/windowsinstaller/bundle/Default.thm b/pypy/tool/release/windowsinstaller/bundle/Default.thm
new file mode 100644
--- /dev/null
+++ b/pypy/tool/release/windowsinstaller/bundle/Default.thm
@@ -0,0 +1,136 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Theme xmlns="http://wixtoolset.org/schemas/thmutil/2010">
+    <Window Width="670" Height="412" HexStyle="100a0000" FontId="0">#(loc.Caption)</Window>
+    <Font Id="0" Height="-14" Weight="500" Foreground="000000" Background="ffffff">Segoe UI</Font>
+    <Font Id="1" Height="-26" Weight="500" Foreground="000000" Background="ffffff">Segoe UI</Font>
+    <Font Id="2" Height="-24" Weight="500" Foreground="808080" Background="ffffff">Segoe UI</Font>
+    <Font Id="3" Height="-14" Weight="500" Foreground="000000" Background="ffffff">Segoe UI</Font>
+    <Font Id="4" Height="-14" Weight="500" Foreground="ff0000" Background="ffffff" Underline="yes">Segoe UI</Font>
+    <Font Id="5" Height="-14" Weight="500" Foreground="808080" Background="ffffff">Segoe UI</Font>
+
+    <Page Name="Help">
+        <Text X="185" Y="11" Width="-11" Height="36" FontId="1" DisablePrefix="yes">#(loc.HelpHeader)</Text>
+        <Image X="0" Y="0" Width="178" Height="382" ImageFile="SideBar.png"/>
+
+        <Hypertext X="185" Y="50" Width="-11" Height="-35" FontId="3" DisablePrefix="yes">#(loc.HelpText)</Hypertext>
+        <Button Name="SuccessCancelButton" X="-11" Y="-11" Width="85" Height="27" TabStop="yes" FontId="0">#(loc.CloseButton)</Button>
+    </Page>
+    <Page Name="Install">
+        <Text X="185" Y="11" Width="-11" Height="36" FontId="1" DisablePrefix="yes">#(loc.InstallHeader)</Text>
+        <Image X="0" Y="0" Width="178" Height="382" ImageFile="SideBar.png"/>
+
+        <Text X="185" Y="50" Width="-11" Height="50" FontId="3" TabStop="yes">#(loc.InstallMessage)</Text>
+
+        <Button Name="InstallButton" X="185" Y="101" Width="-11" Height="109" TabStop="yes" FontId="3" HexStyle="0xE">#(loc.InstallButton)</Button>
+
+        <Checkbox Name="InstallLauncherAllUsers" X="185" Y="-37" Width="-100" Height="24" TabStop="yes" FontId="3">#(loc.ShortInstallLauncherAllUsersLabel)</Checkbox>
+        <Checkbox Name="PrependPath" X="185" Y="-13" Width="-100" Height="24" TabStop="yes" FontId="3">#(loc.ShortPrependPathLabel)</Checkbox>
+
+        <Button Name="InstallCancelButton" X="-11" Y="-11" Width="85" Height="27" TabStop="yes" FontId="0">#(loc.CancelButton)</Button>
+    </Page>
+    <Page Name="Upgrade">
+        <Text X="185" Y="11" Width="-11" Height="36" FontId="1" DisablePrefix="yes">#(loc.InstallUpgradeHeader)</Text>
+        <Image X="0" Y="0" Width="178" Height="382" ImageFile="SideBar.png"/>
+
+        <Text X="185" Y="50" Width="-11" Height="50" FontId="3" TabStop="yes">#(loc.InstallUpgradeMessage)</Text>
+
+        <Button Name="InstallUpgradeButton" X="185" Y="101" Width="-11" Height="129" TabStop="yes" FontId="3" HexStyle="0xE">#(loc.InstallUpgradeButton)</Button>
+        <Button Name="InstallUpgradeCustomButton" X="185" Y="241" Width="-11" Height="59" TabStop="yes" FontId="3" HexStyle="0xE">#(loc.InstallUpgradeCustomButton)</Button>
+
+        <Button Name="InstallCancelButton" X="-11" Y="-11" Width="85" Height="27" TabStop="yes" FontId="0">#(loc.CancelButton)</Button>
+    </Page>
+    <Page Name="SimpleInstall">
+        <Text X="185" Y="11" Width="-11" Height="36" FontId="1" DisablePrefix="yes">#(loc.InstallHeader)</Text>
+        <Image X="0" Y="0" Width="178" Height="382" ImageFile="SideBar.png"/>
+
+        <Button Name="InstallSimpleButton" X="185" Y="101" Width="-11" Height="129" TabStop="yes" FontId="3" HideWhenDisabled="yes" HexStyle="0xF">#(loc.InstallSimpleButton)</Button>
+
+        <Button Name="InstallCancelButton" X="-11" Y="-11" Width="85" Height="27" TabStop="yes" FontId="0">#(loc.CancelButton)</Button>
+    </Page>
+    <Page Name="Custom1">
+        <Text X="185" Y="11" Width="-11" Height="36" FontId="1" DisablePrefix="yes">#(loc.Custom1Header)</Text>
+        <Image X="0" Y="0" Width="178" Height="382" ImageFile="SideBar.png"/>
+        
+        <Checkbox Name="Include_doc" X="185" Y="51" Width="-11" Height="24" TabStop="yes" FontId="3" HideWhenDisabled="yes">#(loc.Include_docLabel)</Checkbox>
+        <Text X="205" Y="76" Width="-11" Height="24" TabStop="no" FontId="5">#(loc.Include_docHelpLabel)</Text>
+        
+        <Checkbox Name="Include_pip" X="185" Y="101" Width="-11" Height="24" TabStop="yes" FontId="3" HideWhenDisabled="yes">#(loc.Include_pipLabel)</Checkbox>
+        <Text X="205" Y="126" Width="-11" Height="24" TabStop="no" FontId="5">#(loc.Include_pipHelpLabel)</Text>
+        
+        <Checkbox Name="Include_tcltk" X="185" Y="151" Width="-11" Height="24" TabStop="yes" FontId="3" HideWhenDisabled="yes">#(loc.Include_tcltkLabel)</Checkbox>
+        <Text X="205" Y="176" Width="-11" Height="24" TabStop="no" FontId="5">#(loc.Include_tcltkHelpLabel)</Text>
+        
+        <Checkbox Name="Include_test" X="185" Y="201" Width="-11" Height="24" TabStop="yes" FontId="3" HideWhenDisabled="yes">#(loc.Include_testLabel)</Checkbox>
+        <Text X="205" Y="226" Width="-11" Height="24" TabStop="no" FontId="5">#(loc.Include_testHelpLabel)</Text>
+
+        <Checkbox Name="Include_launcher" X="185" Y="251" Width="100" Height="24" TabStop="yes" FontId="3" HideWhenDisabled="no">#(loc.Include_launcherLabel)</Checkbox>
+        <Checkbox Name="CustomInstallLauncherAllUsers" X="285" Y="251" Width="-11" Height="24" TabStop="yes" FontId="3">#(loc.InstallLauncherAllUsersLabel)</Checkbox>
+        <Text Name="Include_launcherHelp" X="205" Y="276" Width="-11" Height="24" TabStop="no" FontId="5"></Text>
+
+        <Button Name="Custom1BackButton" X="185" Y="-11" Width="85" Height="27" TabStop="yes" FontId="0">#(loc.CustomBackButton)</Button>
+        <Button Name="CustomNextButton" X="-101" Y="-11" Width="85" Height="27" TabStop="yes" FontId="0">#(loc.CustomNextButton)</Button>
+        <Button Name="Custom1CancelButton" X="-11" Y="-11" Width="85" Height="27" TabStop="yes" FontId="0">#(loc.CancelButton)</Button>
+    </Page>
+    <Page Name="Custom2">
+        <Text X="185" Y="11" Width="-11" Height="36" FontId="1" DisablePrefix="yes">#(loc.Custom2Header)</Text>
+        <Image X="0" Y="0" Width="178" Height="382" ImageFile="SideBar.png"/>
+        
+        <Checkbox Name="InstallAllUsers" X="185" Y="51" Width="-11" Height="24" TabStop="yes" FontId="3" HideWhenDisabled="no">#(loc.InstallAllUsersLabel)</Checkbox>
+        <Checkbox Name="AssociateFiles" X="185" Y="76" Width="-11" Height="24" TabStop="yes" FontId="3" HideWhenDisabled="no">#(loc.AssociateFilesLabel)</Checkbox>
+        <Checkbox Name="Shortcuts" X="185" Y="101" Width="-11" Height="24" TabStop="yes" FontId="3" HideWhenDisabled="no">#(loc.ShortcutsLabel)</Checkbox>
+        <Checkbox Name="PrependPath" X="185" Y="126" Width="-11" Height="24" TabStop="yes" FontId="3" HideWhenDisabled="no">#(loc.PrependPathLabel)</Checkbox>
+        <Checkbox Name="CompileAll" X="185" Y="151" Width="-11" Height="24" TabStop="yes" FontId="3" HideWhenDisabled="no">#(loc.PrecompileLabel)</Checkbox>
+        <Checkbox Name="Include_symbols" X="185" Y="176" Width="-11" Height="24" TabStop="yes" FontId="3" HideWhenDisabled="no">#(loc.Include_symbolsLabel)</Checkbox>
+        <Checkbox Name="Include_debug" X="185" Y="201" Width="-11" Height="24" TabStop="yes" FontId="3" HideWhenDisabled="no">#(loc.Include_debugLabel)</Checkbox>
+
+        <Text X="185" Y="256" Width="-11" Height="17" FontId="3">#(loc.CustomLocationLabel)</Text>
+        <Editbox Name="TargetDir" X="185" Y="277" Width="-101" Height="27" TabStop="yes" FontId="3" FileSystemAutoComplete="yes" />
+        <Button Name="CustomBrowseButton" X="-11" Y="276" Width="85" Height="27" TabStop="yes" FontId="3">#(loc.CustomBrowseButton)</Button>
+        <Text Name="CustomBrowseButtonLabel" X="185" Y="306" Width="-91" Height="35" FontId="5" HideWhenDisabled="yes">#(loc.CustomLocationHelpLabel)</Text>
+
+        <Button Name="Custom2BackButton" X="185" Y="-11" Width="85" Height="27" TabStop="yes" FontId="0">#(loc.CustomBackButton)</Button>
+        <Button Name="CustomInstallButton" X="-101" Y="-11" Width="95" Height="27" TabStop="yes" FontId="0">#(loc.CustomInstallButton)</Button>
+        <Button Name="Custom2CancelButton" X="-11" Y="-11" Width="85" Height="27" TabStop="yes" FontId="0">#(loc.CancelButton)</Button>
+    </Page>
+    <Page Name="Progress">
+        <Text X="185" Y="11" Width="-11" Height="36" FontId="1" DisablePrefix="yes">#(loc.ProgressHeader)</Text>
+        <Image X="0" Y="0" Width="178" Height="382" ImageFile="SideBar.png"/>
+
+        <Text X="185" Y="111" Width="-11" Height="20" FontId="3" DisablePrefix="yes">#(loc.ProgressLabel)</Text>
+        <Text Name="OverallProgressPackageText" X="185" Y="146" Width="-11" Height="20" FontId="3" DisablePrefix="yes">#(loc.OverallProgressPackageText)</Text>
+        <Progressbar Name="OverallCalculatedProgressbar" X="185" Y="171" Width="-11" Height="24" />
+        <Button Name="ProgressCancelButton" X="-11" Y="-11" Width="95" Height="27" TabStop="yes" FontId="0">#(loc.CancelButton)</Button>
+    </Page>
+    <Page Name="Modify">
+        <Text X="185" Y="11" Width="-11" Height="36" FontId="1" DisablePrefix="yes">#(loc.ModifyHeader)</Text>
+        <Image X="0" Y="0" Width="178" Height="382" ImageFile="SideBar.png"/>
+
+        <Button Name="ModifyButton" X="185" Y="101" Width="-11" Height="59" TabStop="yes" FontId="3" HexStyle="0xF">#(loc.ModifyModifyButton)</Button>
+        <Button Name="RepairButton" X="185" Y="171" Width="-11" Height="59" TabStop="yes" FontId="3" HexStyle="0xE">#(loc.ModifyRepairButton)</Button>
+        <Button Name="UninstallButton" X="185" Y="241" Width="-11" Height="59" TabStop="yes" FontId="3" HexStyle="0xE">#(loc.ModifyUninstallButton)</Button>
+
+        <Button Name="ModifyCancelButton" X="-11" Y="-11" Width="85" Height="27" TabStop="yes" FontId="0">#(loc.CancelButton)</Button>
+    </Page>
+    <Page Name="Success">
+        <Text X="185" Y="11" Width="-11" Height="36" FontId="1" DisablePrefix="yes">#(loc.SuccessHeader)</Text>
+        <Image X="0" Y="0" Width="178" Height="382" ImageFile="SideBar.png"/>
+
+        <Hypertext Name="SuccessText" X="205" Y="71" Width="-71" Height="150" FontId="3" DisablePrefix="yes"></Hypertext>
+
+        <Button Name="SuccessMaxPathButton" X="185" Y="-70" Width="-11" Height="81" TabStop="yes" FontId="3" HexStyle="0xE" HideWhenDisabled="yes">#(loc.SuccessMaxPathButton)</Button>
+
+        <Text Name="SuccessRestartText" X="205" Y="-40" Width="-11" Height="34" FontId="3" HideWhenDisabled="yes" DisablePrefix="yes">#(loc.SuccessRestartText)</Text>
+        <Button Name="SuccessRestartButton" X="-101" Y="-11" Width="85" Height="27" TabStop="yes" FontId="0" HideWhenDisabled="yes">#(loc.SuccessRestartButton)</Button>
+        <Button Name="SuccessCancelButton" X="-11" Y="-11" Width="85" Height="27" TabStop="yes" FontId="0">#(loc.CloseButton)</Button>
+    </Page>
+    <Page Name="Failure">
+        <Text X="185" Y="11" Width="-11" Height="36" FontId="1" DisablePrefix="yes">#(loc.FailureHeader)</Text>
+        <Image X="0" Y="0" Width="178" Height="382" ImageFile="SideBar.png"/>
+
+        <Hypertext Name="FailureLogFileLink" X="205" Y="71" Width="-11" Height="60" FontId="3" TabStop="yes" HideWhenDisabled="yes">#(loc.FailureHyperlinkLogText)</Hypertext>
+        <Hypertext Name="FailureMessageText" X="205" Y="151" Width="-11" Height="120" FontId="3" TabStop="yes" HideWhenDisabled="yes"></Hypertext>
+        <Text Name="FailureRestartText" X="205" Y="-40" Width="-11" Height="34" FontId="3" HideWhenDisabled="yes" DisablePrefix="yes">#(loc.FailureRestartText)</Text>
+        <Button Name="FailureRestartButton" X="-101" Y="-11" Width="85" Height="27" TabStop="yes" FontId="0" HideWhenDisabled="yes">#(loc.FailureRestartButton)</Button>
+        <Button Name="FailureCancelButton" X="-11" Y="-11" Width="85" Height="27" TabStop="yes" FontId="0">#(loc.CloseButton)</Button>
+    </Page>
+</Theme>
\ No newline at end of file
diff --git a/pypy/tool/release/windowsinstaller/bundle/Default.wxl b/pypy/tool/release/windowsinstaller/bundle/Default.wxl
new file mode 100644
--- /dev/null
+++ b/pypy/tool/release/windowsinstaller/bundle/Default.wxl
@@ -0,0 +1,145 @@
+&#65279;<?xml version="1.0" encoding="utf-8"?>
+<WixLocalization Culture="en-us" Language="1033" xmlns="http://schemas.microsoft.com/wix/2006/localization">
+  <String Id="Caption">[WixBundleName] Setup</String>
+  <String Id="Title">[WixBundleName]</String>
+  <String Id="Installing">Installing</String>
+  <String Id="Installation">Setup</String>
+  <String Id="Modifying">Updating</String>
+  <String Id="Modification">Modify</String>
+  <String Id="Repairing">Repairing</String>
+  <String Id="Repair">Repair</String>
+  <String Id="Uninstalling">Removing</String>
+  <String Id="Uninstallation">Uninstall</String>
+  
+  <String Id="ElevateForCRTInstall">You will be prompted for Administrator privileges to install a C Runtime Library update (KB2999226).
+
+
+Continue?</String>
+  
+  <String Id="CancelButton">&amp;Cancel</String>
+  <String Id="CloseButton">&amp;Close</String>
+  <String Id="InstallHeader">Install [WixBundleName]</String>
+  <String Id="InstallMessage">Select Install Now to install PyPy with default settings.</String>
+  <String Id="InstallVersion">Version [WixBundleVersion]</String>
+  <String Id="InstallUpgradeHeader">Upgrade to [WixBundleName]</String>
+  <String Id="InstallUpgradeMessage">Select Upgrade Now to keep your current settings, or choose Customize to enable or disable features.</String>
+  <String Id="ConfirmCancelMessage">Are you sure you want to cancel?</String>
+  <String Id="ExecuteUpgradeRelatedBundleMessage">Previous version</String>
+  <String Id="HelpHeader">Setup Help</String>
+  <String Id="HelpText">Visit &lt;a href="http://doc.pypy.org/en/latest/"&gt;doc.pypy.org[ShortVersion]/en/latest/&lt;/a&gt; for the full list of options, including the ability to enable and disable specific features.
+
+"/passive" to display progress without requiring user interaction
+
+"/quiet" to install/uninstall without displaying any UI
+
+"/simple" to prevent user customization
+
+"/uninstall" to remove PyPy (without confirmation)
+
+"/layout [\[]directory[\]]" to pre-download all components
+
+"/log [\[]filename[\]]" to specify log files location</String>
+  <String Id="InstallLicenseLinkText">[WixBundleName] &lt;a href="#"&gt;license terms&lt;/a&gt;.</String>
+  <String Id="InstallAcceptCheckbox">I &amp;agree to the license terms and conditions</String>
+  <String Id="InstallButton">&amp;Install Now</String>
+  <String Id="InstallButtonNote">[TargetDir]
+
+Creates shortcuts and file associations</String>
+  <String Id="InstallCustomButton">C&amp;ustomize installation</String>
+  <String Id="InstallCustomButtonNote">Choose location and features</String>
+  <String Id="InstallSimpleButton">&amp;Install</String>
+  <String Id="InstallSimpleButtonNote">Use settings preselected by your administrator
+
+[SimpleInstallDescription]</String>
+  <String Id="InstallUpgradeButton">Up&amp;grade Now</String>
+  <String Id="InstallUpgradeButtonNote">[TargetDir]
+
+Replaces your existing installation without changing settings.
+Select Customize to review current options.</String>
+  <String Id="InstallUpgradeCustomButton">C&amp;ustomize installation</String>
+  <String Id="InstallUpgradeCustomButtonNote">Choose location and features</String>
+  <String Id="Custom1Header">Optional Features</String>
+  <String Id="Custom2Header">Advanced Options</String>
+  <String Id="CustomLocationLabel">Customize install location</String>
+  <String Id="CustomLocationHelpLabel">You will require write permissions for the selected location.</String>
+  <String Id="CustomInstallButton">&amp;Install</String>
+  <String Id="CustomNextButton">&amp;Next</String>
+  <String Id="CustomBackButton">&amp;Back</String>
+  <String Id="CustomBrowseButton">B&amp;rowse</String>
+  <String Id="Include_docLabel">&amp;Documentation</String>
+  <String Id="Include_docHelpLabel">Installs the PyPy documentation file.</String>
+  <String Id="Include_pipLabel">&amp;pip</String>
+  <String Id="Include_pipHelpLabel">Installs pip, which can download and install other PyPy packages.</String>
+  <String Id="Include_tcltkLabel">tcl/tk and &amp;IDLE</String>
+  <String Id="Include_tcltkHelpLabel">Installs tkinter and the IDLE development environment.</String>
+  <String Id="Include_testLabel">PyPy &amp;test suite</String>
+  <String Id="Include_testHelpLabel">Installs the standard library test suite.</String>
+  <String Id="Include_launcherLabel">py &amp;launcher</String>
+  <String Id="Include_launcherHelp">Installs the global 'py' launcher to make it easier to start Python.</String>
+  <String Id="Include_launcherRemove">Use Programs and Features to remove the 'py' launcher.</String>
+  <String Id="Include_launcherUpgrade">Upgrades the global 'py' launcher from the previous version.</String>
+  
+  <String Id="AssociateFilesLabel">Associate &amp;files with PyPy (requires the py launcher)</String>
+  <String Id="ShortcutsLabel">Create shortcuts for installed applications</String>
+  <String Id="PrependPathLabel">Add PyPy to &amp;environment variables</String>
+  <String Id="ShortPrependPathLabel">Add &amp;PyPy [ShortVersion] to PATH</String>
+  <String Id="InstallAllUsersLabel">Install for &amp;all users</String>
+  <String Id="InstallLauncherAllUsersLabel">for &amp;all users (requires elevation)</String>
+  <String Id="ShortInstallLauncherAllUsersLabel">Install &amp;launcher for all users (recommended)</String>
+  <String Id="PrecompileLabel">&amp;Precompile standard library</String>
+  <String Id="Include_symbolsLabel">Download debugging &amp;symbols</String>
+  <String Id="Include_debugLabel">Download debu&amp;g binaries (requires VS 2015 or later)</String>
+  
+  <String Id="ProgressHeader">[ActionLikeInstallation] Progress</String>
+  <String Id="ProgressLabel">[ActionLikeInstalling]:</String>
+  <String Id="OverallProgressPackageText">Initializing...</String>
+  <String Id="ModifyHeader">Modify Setup</String>
+  <String Id="ModifyModifyButton">&amp;Modify</String>
+  <String Id="ModifyButtonNote">Add or remove individual features.</String>
+  <String Id="ModifyRepairButton">&amp;Repair</String>
+  <String Id="RepairButtonNote">Ensure all current features are correctly installed.</String>
+  <String Id="ModifyUninstallButton">&amp;Uninstall</String>
+  <String Id="UninstallButtonNote">Remove the entire [WixBundleName] installation.</String>
+  <String Id="SuccessHeader">[ActionLikeInstallation] was successful</String>
+  <String Id="SuccessLaunchButton">&amp;Launch</String>
+  <String Id="SuccessRestartText">You may need to restart your computer to finish updating files.</String>
+  <String Id="SuccessRestartButton">&amp;Restart</String>
+  <String Id="SuccessInstallMessage">New to Python? Start with the &lt;a href="https://docs.python.org/[ShortVersion]/tutorial/index.html"&gt;online tutorial&lt;/a&gt; and &lt;a href="https://docs.python.org/[ShortVersion]/index.html"&gt;documentation&lt;/a&gt;.
+
+See &lt;a href="https://docs.python.org/[ShortVersion]/whatsnew/[ShortVersion].html"&gt;what's new&lt;/a&gt; in this release.</String>
+  <String Id="SuccessModifyMessage">Thank you for using [WixBundleName].</String>
+  <String Id="SuccessRepairMessage">Thank you for using [WixBundleName].
+
+Feel free to email &lt;a href="mailto:pypy-dev at python.org"&gt;pypy-dev at python.org&lt;/a&gt; if you continue to encounter issues.</String>
+  <String Id="SuccessRemoveMessage">Thank you for using [WixBundleName].
+
+Feel free to email &lt;a href="mailto:pypy-dev at python.org"&gt;pypy-dev at python.org&lt;/a&gt; if you encountered problems.</String>
+  <String Id="FailureHeader">Setup failed</String>
+  <String Id="FailureHyperlinkLogText">One or more issues caused the setup to fail. Please fix the issues and then retry setup. For more information see the &lt;a href="#"&gt;log file&lt;/a&gt;.</String>
+  <String Id="FailureRestartText">You must restart your computer to complete the rollback of the software.</String>
+  <String Id="FailureRestartButton">&amp;Restart</String>
+  <String Id="FailureExistingInstall">Unable to install [WixBundleName] due to an existing install. Use Programs and Features to modify, repair or remove [WixBundleName].</String>
+  
+  <String Id="FailureWin7MissingSP1">Windows 7 Service Pack 1 and all applicable updates are required to install [WixBundleName].
+
+Please &lt;a href="https://www.bing.com/search?q=how%20to%20install%20windows%207%20service%20pack%201"&gt;update your machine&lt;/a&gt; and then restart the installation.</String>
+  <String Id="FailureVistaMissingSP2">Windows Vista Service Pack 2 and all applicable updates are required to install [WixBundleName].
+
+Please &lt;a href="https://www.bing.com/search?q=how%20to%20install%20windows%20vista%20service%20pack%202"&gt;update your machine&lt;/a&gt; and then restart the installation.</String>
+  <String Id="FailureXPOrEarlier">Windows Vista or later is required to install and use [WixBundleName].
+
+Visit &lt;a href="https://www.python.org/"&gt;python.org&lt;/a&gt; to download Python 3.4.</String>
+
+  <String Id="FailureWS2K8R2MissingSP1">Windows Server 2008 R2 Service Pack 1 and all applicable updates are required to install [WixBundleName].
+
+Please &lt;a href="https://www.bing.com/search?q=how%20to%20install%20windows%20server%202008%20r2%20service%20pack%201"&gt;update your machine&lt;/a&gt; and then restart the installation.</String>
+  <String Id="FailureWS2K8MissingSP2">Windows Server 2008 Service Pack 2 and all applicable updates are required to install [WixBundleName].
+
+Please &lt;a href="https://www.bing.com/search?q=how%20to%20install%20windows%20server%202008%20service%20pack%202"&gt;update your machine&lt;/a&gt; and then restart the installation.</String>
+  <String Id="FailureWS2K3OrEarlier">Windows Server 2008 SP2 or later is required to install and use [WixBundleName].
+
+Visit &lt;a href="https://www.python.org/"&gt;python.org&lt;/a&gt; to download Python 3.4.</String>
+
+  <String Id="SuccessMaxPathButton">Disable path length limit</String>
+  <String Id="SuccessMaxPathButtonNote">Changes your machine configuration to allow programs, including Python, to bypass the 260 character "MAX_PATH" limitation.</String>
+</WixLocalization>
diff --git a/pypy/tool/release/windowsinstaller/bundle/SideBar.png b/pypy/tool/release/windowsinstaller/bundle/SideBar.png
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..037da0d8515b469ff9e6ef1ad0e9b393b6f1f154
GIT binary patch

[cut]

diff --git a/pypy/tool/release/windowsinstaller/bundle/SideBar.xcf b/pypy/tool/release/windowsinstaller/bundle/SideBar.xcf
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..19f731cb5a7d8be36ef5f14d5dfd677058ffeb75
GIT binary patch

[cut]

diff --git a/pypy/tool/release/windowsinstaller/bundle/bootstrap/LICENSE.txt b/pypy/tool/release/windowsinstaller/bundle/bootstrap/LICENSE.txt
new file mode 100644
--- /dev/null
+++ b/pypy/tool/release/windowsinstaller/bundle/bootstrap/LICENSE.txt
@@ -0,0 +1,25 @@
+This license applies to the bootstrapper application that is embedded within the installer. It has no impact on the licensing for the rest of the installer or Python itself, as no code covered by this license exists in any other part of the product.
+
+---
+
+Microsoft Reciprocal License (MS-RL)
+
+This license governs use of the accompanying software. If you use the software, you accept this license. If you do not accept the license, do not use the software.
+
+1. Definitions
+ The terms "reproduce," "reproduction," "derivative works," and "distribution" have the same meaning here as under U.S. copyright law.
+ A "contribution" is the original software, or any additions or changes to the software.
+ A "contributor" is any person that distributes its contribution under this license.
+ "Licensed patents" are a contributor's patent claims that read directly on its contribution.
+
+2. Grant of Rights
+ (A) Copyright Grant- Subject to the terms of this license, including the license conditions and limitations in section 3, each contributor grants you a non-exclusive, worldwide, royalty-free copyright license to reproduce its contribution, prepare derivative works of its contribution, and distribute its contribution or any derivative works that you create.
+ (B) Patent Grant- Subject to the terms of this license, including the license conditions and limitations in section 3, each contributor grants you a non-exclusive, worldwide, royalty-free license under its licensed patents to make, have made, use, sell, offer for sale, import, and/or otherwise dispose of its contribution in the software or derivative works of the contribution in the software.
+
+3. Conditions and Limitations
+ (A) Reciprocal Grants- For any file you distribute that contains code from the software (in source code or binary format), you must provide recipients the source code to that file along with a copy of this license, which license will govern that file. You may license other files that are entirely your own work and do not contain code from the software under any terms you choose.
+ (B) No Trademark License- This license does not grant you rights to use any contributors' name, logo, or trademarks.
+ (C) If you bring a patent claim against any contributor over patents that you claim are infringed by the software, your patent license from such contributor to the software ends automatically.
+ (D) If you distribute any portion of the software, you must retain all copyright, patent, trademark, and attribution notices that are present in the software.
+ (E) If you distribute any portion of the software in source code form, you may do so only under this license by including a complete copy of this license with your distribution. If you distribute any portion of the software in compiled or object code form, you may only do so under a license that complies with this license.
+ (F) The software is licensed "as-is." You bear the risk of using it. The contributors give no express warranties, guarantees or conditions. You may have additional consumer rights under your local laws which this license cannot change. To the extent permitted under your local laws, the contributors exclude the implied warranties of merchantability, fitness for a particular purpose and non-infringement.
diff --git a/pypy/tool/release/windowsinstaller/bundle/bootstrap/PythonBootstrapperApplication.cpp b/pypy/tool/release/windowsinstaller/bundle/bootstrap/PythonBootstrapperApplication.cpp
new file mode 100644
--- /dev/null
+++ b/pypy/tool/release/windowsinstaller/bundle/bootstrap/PythonBootstrapperApplication.cpp
@@ -0,0 +1,3233 @@
+//-------------------------------------------------------------------------------------------------
+// <copyright file="WixStandardBootstrapperApplication.cpp" company="Outercurve Foundation">
+//   Copyright (c) 2004, Outercurve Foundation.
+//   This software is released under Microsoft Reciprocal License (MS-RL).
+//   The license and further copyright text can be found in the file
+//   LICENSE.TXT at the root directory of the distribution.
+// </copyright>
+//-------------------------------------------------------------------------------------------------
+
+
+#include "pch.h"
+
+static const LPCWSTR PYBA_WINDOW_CLASS = L"PythonBA";
+static const DWORD PYBA_ACQUIRE_PERCENTAGE = 30;
+static const LPCWSTR PYBA_VARIABLE_BUNDLE_FILE_VERSION = L"WixBundleFileVersion";
+
+enum PYBA_STATE {
+    PYBA_STATE_INITIALIZING,
+    PYBA_STATE_INITIALIZED,
+    PYBA_STATE_HELP,
+    PYBA_STATE_DETECTING,
+    PYBA_STATE_DETECTED,
+    PYBA_STATE_PLANNING,
+    PYBA_STATE_PLANNED,
+    PYBA_STATE_APPLYING,
+    PYBA_STATE_CACHING,
+    PYBA_STATE_CACHED,
+    PYBA_STATE_EXECUTING,
+    PYBA_STATE_EXECUTED,
+    PYBA_STATE_APPLIED,
+    PYBA_STATE_FAILED,
+};
+
+static const int WM_PYBA_SHOW_HELP = WM_APP + 100;
+static const int WM_PYBA_DETECT_PACKAGES = WM_APP + 101;
+static const int WM_PYBA_PLAN_PACKAGES = WM_APP + 102;
+static const int WM_PYBA_APPLY_PACKAGES = WM_APP + 103;
+static const int WM_PYBA_CHANGE_STATE = WM_APP + 104;
+static const int WM_PYBA_SHOW_FAILURE = WM_APP + 105;
+
+// This enum must be kept in the same order as the PAGE_NAMES array.
+enum PAGE {
+    PAGE_LOADING,
+    PAGE_HELP,
+    PAGE_INSTALL,
+    PAGE_UPGRADE,
+    PAGE_SIMPLE_INSTALL,
+    PAGE_CUSTOM1,
+    PAGE_CUSTOM2,
+    PAGE_MODIFY,
+    PAGE_PROGRESS,
+    PAGE_PROGRESS_PASSIVE,
+    PAGE_SUCCESS,
+    PAGE_FAILURE,
+    COUNT_PAGE,
+};
+
+// This array must be kept in the same order as the PAGE enum.
+static LPCWSTR PAGE_NAMES[] = {
+    L"Loading",
+    L"Help",
+    L"Install",
+    L"Upgrade",
+    L"SimpleInstall",
+    L"Custom1",
+    L"Custom2",
+    L"Modify",
+    L"Progress",
+    L"ProgressPassive",
+    L"Success",
+    L"Failure",
+};
+
+enum CONTROL_ID {
+    // Non-paged controls
+    ID_CLOSE_BUTTON = THEME_FIRST_ASSIGN_CONTROL_ID,
+    ID_MINIMIZE_BUTTON,
+
+    // Welcome page
+    ID_INSTALL_BUTTON,
+    ID_INSTALL_CUSTOM_BUTTON,
+    ID_INSTALL_SIMPLE_BUTTON,
+    ID_INSTALL_UPGRADE_BUTTON,
+    ID_INSTALL_UPGRADE_CUSTOM_BUTTON,
+    ID_INSTALL_CANCEL_BUTTON,
+    ID_INSTALL_LAUNCHER_ALL_USERS_CHECKBOX,
+
+    // Customize Page
+    ID_TARGETDIR_EDITBOX,
+    ID_CUSTOM_ASSOCIATE_FILES_CHECKBOX,
+    ID_CUSTOM_INSTALL_ALL_USERS_CHECKBOX,
+    ID_CUSTOM_INSTALL_LAUNCHER_ALL_USERS_CHECKBOX,
+    ID_CUSTOM_INCLUDE_LAUNCHER_HELP_LABEL,
+    ID_CUSTOM_COMPILE_ALL_CHECKBOX,
+    ID_CUSTOM_BROWSE_BUTTON,
+    ID_CUSTOM_BROWSE_BUTTON_LABEL,
+    ID_CUSTOM_INSTALL_BUTTON,
+    ID_CUSTOM_NEXT_BUTTON,
+    ID_CUSTOM1_BACK_BUTTON,
+    ID_CUSTOM2_BACK_BUTTON,
+    ID_CUSTOM1_CANCEL_BUTTON,
+    ID_CUSTOM2_CANCEL_BUTTON,
+
+    // Modify page
+    ID_MODIFY_BUTTON,
+    ID_REPAIR_BUTTON,
+    ID_UNINSTALL_BUTTON,
+    ID_MODIFY_CANCEL_BUTTON,
+
+    // Progress page
+    ID_CACHE_PROGRESS_PACKAGE_TEXT,
+    ID_CACHE_PROGRESS_BAR,
+    ID_CACHE_PROGRESS_TEXT,
+
+    ID_EXECUTE_PROGRESS_PACKAGE_TEXT,
+    ID_EXECUTE_PROGRESS_BAR,
+    ID_EXECUTE_PROGRESS_TEXT,
+    ID_EXECUTE_PROGRESS_ACTIONDATA_TEXT,
+
+    ID_OVERALL_PROGRESS_PACKAGE_TEXT,
+    ID_OVERALL_PROGRESS_BAR,
+    ID_OVERALL_CALCULATED_PROGRESS_BAR,
+    ID_OVERALL_PROGRESS_TEXT,
+
+    ID_PROGRESS_CANCEL_BUTTON,
+
+    // Success page
+    ID_SUCCESS_TEXT,
+    ID_SUCCESS_RESTART_TEXT,
+    ID_SUCCESS_RESTART_BUTTON,
+    ID_SUCCESS_CANCEL_BUTTON,
+    ID_SUCCESS_MAX_PATH_BUTTON,
+
+    // Failure page
+    ID_FAILURE_LOGFILE_LINK,
+    ID_FAILURE_MESSAGE_TEXT,
+    ID_FAILURE_RESTART_TEXT,
+    ID_FAILURE_RESTART_BUTTON,
+    ID_FAILURE_CANCEL_BUTTON
+};
+
+static THEME_ASSIGN_CONTROL_ID CONTROL_ID_NAMES[] = {
+    { ID_CLOSE_BUTTON, L"CloseButton" },
+    { ID_MINIMIZE_BUTTON, L"MinimizeButton" },
+
+    { ID_INSTALL_BUTTON, L"InstallButton" },
+    { ID_INSTALL_CUSTOM_BUTTON, L"InstallCustomButton" },
+    { ID_INSTALL_SIMPLE_BUTTON, L"InstallSimpleButton" },
+    { ID_INSTALL_UPGRADE_BUTTON, L"InstallUpgradeButton" },
+    { ID_INSTALL_UPGRADE_CUSTOM_BUTTON, L"InstallUpgradeCustomButton" },
+    { ID_INSTALL_CANCEL_BUTTON, L"InstallCancelButton" },
+    { ID_INSTALL_LAUNCHER_ALL_USERS_CHECKBOX, L"InstallLauncherAllUsers" },
+
+    { ID_TARGETDIR_EDITBOX, L"TargetDir" },
+    { ID_CUSTOM_ASSOCIATE_FILES_CHECKBOX, L"AssociateFiles" },
+    { ID_CUSTOM_INSTALL_ALL_USERS_CHECKBOX, L"InstallAllUsers" },
+    { ID_CUSTOM_INSTALL_LAUNCHER_ALL_USERS_CHECKBOX, L"CustomInstallLauncherAllUsers" },
+    { ID_CUSTOM_INCLUDE_LAUNCHER_HELP_LABEL, L"Include_launcherHelp" },
+    { ID_CUSTOM_COMPILE_ALL_CHECKBOX, L"CompileAll" },
+    { ID_CUSTOM_BROWSE_BUTTON, L"CustomBrowseButton" },
+    { ID_CUSTOM_BROWSE_BUTTON_LABEL, L"CustomBrowseButtonLabel" },
+    { ID_CUSTOM_INSTALL_BUTTON, L"CustomInstallButton" },
+    { ID_CUSTOM_NEXT_BUTTON, L"CustomNextButton" },
+    { ID_CUSTOM1_BACK_BUTTON, L"Custom1BackButton" },
+    { ID_CUSTOM2_BACK_BUTTON, L"Custom2BackButton" },
+    { ID_CUSTOM1_CANCEL_BUTTON, L"Custom1CancelButton" },
+    { ID_CUSTOM2_CANCEL_BUTTON, L"Custom2CancelButton" },
+
+    { ID_MODIFY_BUTTON, L"ModifyButton" },
+    { ID_REPAIR_BUTTON, L"RepairButton" },
+    { ID_UNINSTALL_BUTTON, L"UninstallButton" },
+    { ID_MODIFY_CANCEL_BUTTON, L"ModifyCancelButton" },
+
+    { ID_CACHE_PROGRESS_PACKAGE_TEXT, L"CacheProgressPackageText" },
+    { ID_CACHE_PROGRESS_BAR, L"CacheProgressbar" },
+    { ID_CACHE_PROGRESS_TEXT, L"CacheProgressText" },
+    { ID_EXECUTE_PROGRESS_PACKAGE_TEXT, L"ExecuteProgressPackageText" },
+    { ID_EXECUTE_PROGRESS_BAR, L"ExecuteProgressbar" },
+    { ID_EXECUTE_PROGRESS_TEXT, L"ExecuteProgressText" },
+    { ID_EXECUTE_PROGRESS_ACTIONDATA_TEXT, L"ExecuteProgressActionDataText" },
+    { ID_OVERALL_PROGRESS_PACKAGE_TEXT, L"OverallProgressPackageText" },
+    { ID_OVERALL_PROGRESS_BAR, L"OverallProgressbar" },
+    { ID_OVERALL_CALCULATED_PROGRESS_BAR, L"OverallCalculatedProgressbar" },
+    { ID_OVERALL_PROGRESS_TEXT, L"OverallProgressText" },
+    { ID_PROGRESS_CANCEL_BUTTON, L"ProgressCancelButton" },
+
+    { ID_SUCCESS_TEXT, L"SuccessText" },
+    { ID_SUCCESS_RESTART_TEXT, L"SuccessRestartText" },
+    { ID_SUCCESS_RESTART_BUTTON, L"SuccessRestartButton" },
+    { ID_SUCCESS_CANCEL_BUTTON, L"SuccessCancelButton" },
+    { ID_SUCCESS_MAX_PATH_BUTTON, L"SuccessMaxPathButton" },
+
+    { ID_FAILURE_LOGFILE_LINK, L"FailureLogFileLink" },
+    { ID_FAILURE_MESSAGE_TEXT, L"FailureMessageText" },
+    { ID_FAILURE_RESTART_TEXT, L"FailureRestartText" },
+    { ID_FAILURE_RESTART_BUTTON, L"FailureRestartButton" },
+    { ID_FAILURE_CANCEL_BUTTON, L"FailureCancelButton" },
+};
+
+static struct { LPCWSTR regName; LPCWSTR variableName; } OPTIONAL_FEATURES[] = {
+    { L"core_d", L"Include_debug" },
+    { L"core_pdb", L"Include_symbols" },
+    { L"dev", L"Include_dev" },
+    { L"doc", L"Include_doc" },
+    { L"exe", L"Include_exe" },
+    { L"lib", L"Include_lib" },
+    { L"path", L"PrependPath" },
+    { L"pip", L"Include_pip" },
+    { L"tcltk", L"Include_tcltk" },
+    { L"test", L"Include_test" },
+    { L"tools", L"Include_tools" },
+    { L"Shortcuts", L"Shortcuts" },
+    // Include_launcher and AssociateFiles are handled separately and so do
+    // not need to be included in this list.
+    { nullptr, nullptr }
+};
+
+
+
+class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication {
+    void ShowPage(DWORD newPageId) {
+        // Process each control for special handling in the new page.
+        ProcessPageControls(ThemeGetPage(_theme, newPageId));
+
+        // Enable disable controls per-page.
+        if (_pageIds[PAGE_INSTALL] == newPageId ||
+            _pageIds[PAGE_SIMPLE_INSTALL] == newPageId ||
+            _pageIds[PAGE_UPGRADE] == newPageId) {
+            InstallPage_Show();
+        } else if (_pageIds[PAGE_CUSTOM1] == newPageId) {
+            Custom1Page_Show();
+        } else if (_pageIds[PAGE_CUSTOM2] == newPageId) {
+            Custom2Page_Show();
+        } else if (_pageIds[PAGE_MODIFY] == newPageId) {
+            ModifyPage_Show();
+        } else if (_pageIds[PAGE_SUCCESS] == newPageId) {
+            SuccessPage_Show();
+        } else if (_pageIds[PAGE_FAILURE] == newPageId) {
+            FailurePage_Show();
+        }
+
+        // Prevent repainting while switching page to avoid ugly flickering
+        _suppressPaint = TRUE;
+        ThemeShowPage(_theme, newPageId, SW_SHOW);
+        ThemeShowPage(_theme, _visiblePageId, SW_HIDE);
+        _suppressPaint = FALSE;
+        InvalidateRect(_theme->hwndParent, nullptr, TRUE);
+        _visiblePageId = newPageId;
+
+        // On the install page set the focus to the install button or
+        // the next enabled control if install is disabled
+        if (_pageIds[PAGE_INSTALL] == newPageId) {
+            ThemeSetFocus(_theme, ID_INSTALL_BUTTON);
+        } else if (_pageIds[PAGE_SIMPLE_INSTALL] == newPageId) {
+            ThemeSetFocus(_theme, ID_INSTALL_SIMPLE_BUTTON);
+        }
+    }
+
+    //
+    // Handles control clicks
+    //
+    void OnCommand(CONTROL_ID id) {
+        LPWSTR defaultDir = nullptr;
+        LPWSTR targetDir = nullptr;
+        LONGLONG elevated, crtInstalled, installAllUsers;
+        BOOL checked, launcherChecked;
+        WCHAR wzPath[MAX_PATH] = { };
+        BROWSEINFOW browseInfo = { };
+        PIDLIST_ABSOLUTE pidl = nullptr;
+        DWORD pageId;
+        HRESULT hr = S_OK;
+
+        switch(id) {
+        case ID_CLOSE_BUTTON:
+            OnClickCloseButton();
+            break;
+
+        // Install commands
+        case ID_INSTALL_SIMPLE_BUTTON: __fallthrough;
+        case ID_INSTALL_UPGRADE_BUTTON: __fallthrough;
+        case ID_INSTALL_BUTTON:
+            SavePageSettings();
+
+            hr = BalGetNumericVariable(L"InstallAllUsers", &installAllUsers);
+            ExitOnFailure(hr, L"Failed to get install scope");
+
+            hr = _engine->SetVariableNumeric(L"CompileAll", installAllUsers);
+            ExitOnFailure(hr, L"Failed to update CompileAll");
+
+            hr = EnsureTargetDir();
+            ExitOnFailure(hr, L"Failed to set TargetDir");
+
+            OnPlan(BOOTSTRAPPER_ACTION_INSTALL);
+            break;
+
+        case ID_CUSTOM1_BACK_BUTTON:
+            SavePageSettings();
+            if (_modifying) {
+                GoToPage(PAGE_MODIFY);
+            } else if (_upgrading) {
+                GoToPage(PAGE_UPGRADE);
+            } else {
+                GoToPage(PAGE_INSTALL);
+            }
+            break;
+
+        case ID_INSTALL_CUSTOM_BUTTON: __fallthrough;
+        case ID_INSTALL_UPGRADE_CUSTOM_BUTTON: __fallthrough;
+        case ID_CUSTOM2_BACK_BUTTON:
+            SavePageSettings();
+            GoToPage(PAGE_CUSTOM1);
+            break;
+
+        case ID_CUSTOM_NEXT_BUTTON:
+            SavePageSettings();
+            GoToPage(PAGE_CUSTOM2);
+            break;
+
+        case ID_CUSTOM_INSTALL_BUTTON:
+            SavePageSettings();
+
+            hr = EnsureTargetDir();
+            ExitOnFailure(hr, L"Failed to set TargetDir");
+
+            hr = BalGetStringVariable(L"TargetDir", &targetDir);
+            if (SUCCEEDED(hr)) {
+                // TODO: Check whether directory exists and contains another installation
+                ReleaseStr(targetDir);
+            }
+
+            OnPlan(_command.action);
+            break;
+
+        case ID_INSTALL_LAUNCHER_ALL_USERS_CHECKBOX:
+            checked = ThemeIsControlChecked(_theme, ID_INSTALL_LAUNCHER_ALL_USERS_CHECKBOX);
+            _engine->SetVariableNumeric(L"InstallLauncherAllUsers", checked);
+
+            ThemeControlElevates(_theme, ID_INSTALL_BUTTON, WillElevate());
+            break;
+
+        case ID_CUSTOM_INSTALL_LAUNCHER_ALL_USERS_CHECKBOX:
+            checked = ThemeIsControlChecked(_theme, ID_CUSTOM_INSTALL_LAUNCHER_ALL_USERS_CHECKBOX);
+            _engine->SetVariableNumeric(L"InstallLauncherAllUsers", checked);
+
+            ThemeControlElevates(_theme, ID_CUSTOM_INSTALL_BUTTON, WillElevate());
+            break;
+
+        case ID_CUSTOM_INSTALL_ALL_USERS_CHECKBOX:
+            checked = ThemeIsControlChecked(_theme, ID_CUSTOM_INSTALL_ALL_USERS_CHECKBOX);
+            _engine->SetVariableNumeric(L"InstallAllUsers", checked);
+
+            ThemeControlElevates(_theme, ID_CUSTOM_INSTALL_BUTTON, WillElevate());
+            ThemeControlEnable(_theme, ID_CUSTOM_BROWSE_BUTTON_LABEL, !checked);
+            if (checked) {
+                _engine->SetVariableNumeric(L"CompileAll", 1);
+                ThemeSendControlMessage(_theme, ID_CUSTOM_COMPILE_ALL_CHECKBOX, BM_SETCHECK, BST_CHECKED, 0);
+            }
+            ThemeGetTextControl(_theme, ID_TARGETDIR_EDITBOX, &targetDir);
+            if (targetDir) {
+                // Check the current value against the default to see
+                // if we should switch it automatically.
+                hr = BalGetStringVariable(
+                    checked ? L"DefaultJustForMeTargetDir" : L"DefaultAllUsersTargetDir",
+                    &defaultDir
+                );
+                
+                if (SUCCEEDED(hr) && defaultDir) {
+                    LPWSTR formatted = nullptr;
+                    if (defaultDir[0] && SUCCEEDED(BalFormatString(defaultDir, &formatted))) {
+                        if (wcscmp(formatted, targetDir) == 0) {
+                            ReleaseStr(defaultDir);
+                            defaultDir = nullptr;
+                            ReleaseStr(formatted);
+                            formatted = nullptr;
+                            
+                            hr = BalGetStringVariable(
+                                checked ? L"DefaultAllUsersTargetDir" : L"DefaultJustForMeTargetDir",
+                                &defaultDir
+                            );
+                            if (SUCCEEDED(hr) && defaultDir && defaultDir[0] && SUCCEEDED(BalFormatString(defaultDir, &formatted))) {
+                                ThemeSetTextControl(_theme, ID_TARGETDIR_EDITBOX, formatted);
+                                ReleaseStr(formatted);
+                            }
+                        } else {
+                            ReleaseStr(formatted);
+                        }
+                    }
+                    
+                    ReleaseStr(defaultDir);
+                }
+            }
+            break;
+
+        case ID_CUSTOM_BROWSE_BUTTON:
+            browseInfo.hwndOwner = _hWnd;
+            browseInfo.pszDisplayName = wzPath;
+            browseInfo.lpszTitle = _theme->sczCaption;
+            browseInfo.ulFlags = BIF_RETURNONLYFSDIRS | BIF_USENEWUI;
+            pidl = ::SHBrowseForFolderW(&browseInfo);
+            if (pidl && ::SHGetPathFromIDListW(pidl, wzPath)) {
+                ThemeSetTextControl(_theme, ID_TARGETDIR_EDITBOX, wzPath);
+            }
+
+            if (pidl) {
+                ::CoTaskMemFree(pidl);
+            }
+            break;
+
+        // Modify commands
+        case ID_MODIFY_BUTTON:
+            // Some variables cannot be modified
+            _engine->SetVariableString(L"InstallAllUsersState", L"disable");
+            _engine->SetVariableString(L"InstallLauncherAllUsersState", L"disable");
+            _engine->SetVariableString(L"TargetDirState", L"disable");
+            _engine->SetVariableString(L"CustomBrowseButtonState", L"disable");
+            _modifying = TRUE;
+            GoToPage(PAGE_CUSTOM1);
+            break;
+
+        case ID_REPAIR_BUTTON:
+            OnPlan(BOOTSTRAPPER_ACTION_REPAIR);
+            break;
+
+        case ID_UNINSTALL_BUTTON:
+            OnPlan(BOOTSTRAPPER_ACTION_UNINSTALL);
+            break;
+
+        case ID_SUCCESS_MAX_PATH_BUTTON:
+            EnableMaxPathSupport();
+            ThemeControlEnable(_theme, ID_SUCCESS_MAX_PATH_BUTTON, FALSE);
+            break;
+        }
+
+    LExit:
+        return;
+    }
+
+    void InstallPage_Show() {
+        // Ensure the All Users install button has a UAC shield
+        BOOL elevated = WillElevate();
+        ThemeControlElevates(_theme, ID_INSTALL_BUTTON, elevated);
+        ThemeControlElevates(_theme, ID_INSTALL_SIMPLE_BUTTON, elevated);
+        ThemeControlElevates(_theme, ID_INSTALL_UPGRADE_BUTTON, elevated);
+    }
+
+    void Custom1Page_Show() {
+        LONGLONG installLauncherAllUsers;
+
+        if (FAILED(BalGetNumericVariable(L"InstallLauncherAllUsers", &installLauncherAllUsers))) {
+            installLauncherAllUsers = 0;
+        }
+
+        ThemeSendControlMessage(_theme, ID_CUSTOM_INSTALL_LAUNCHER_ALL_USERS_CHECKBOX, BM_SETCHECK,
+            installLauncherAllUsers ? BST_CHECKED : BST_UNCHECKED, 0);
+
+        LOC_STRING *pLocString = nullptr;
+        LPCWSTR locKey = L"#(loc.Include_launcherHelp)";
+        LONGLONG detectedLauncher;
+
+        if (SUCCEEDED(BalGetNumericVariable(L"DetectedLauncher", &detectedLauncher)) && detectedLauncher) {
+            locKey = L"#(loc.Include_launcherRemove)";
+        } else if (SUCCEEDED(BalGetNumericVariable(L"DetectedOldLauncher", &detectedLauncher)) && detectedLauncher) {
+            locKey = L"#(loc.Include_launcherUpgrade)";
+        }
+
+        if (SUCCEEDED(LocGetString(_wixLoc, locKey, &pLocString)) && pLocString) {
+            ThemeSetTextControl(_theme, ID_CUSTOM_INCLUDE_LAUNCHER_HELP_LABEL, pLocString->wzText);
+        }
+    }
+
+    void Custom2Page_Show() {
+        HRESULT hr;
+        LONGLONG installAll, includeLauncher;
+        
+        if (FAILED(BalGetNumericVariable(L"InstallAllUsers", &installAll))) {
+            installAll = 0;
+        }
+
+        if (WillElevate()) {
+            ThemeControlElevates(_theme, ID_CUSTOM_INSTALL_BUTTON, TRUE);
+            ThemeShowControl(_theme, ID_CUSTOM_BROWSE_BUTTON_LABEL, SW_HIDE);
+        } else {
+            ThemeControlElevates(_theme, ID_CUSTOM_INSTALL_BUTTON, FALSE);
+            ThemeShowControl(_theme, ID_CUSTOM_BROWSE_BUTTON_LABEL, SW_SHOW);
+        }
+
+        if (SUCCEEDED(BalGetNumericVariable(L"Include_launcher", &includeLauncher)) && includeLauncher) {
+            ThemeControlEnable(_theme, ID_CUSTOM_ASSOCIATE_FILES_CHECKBOX, TRUE);
+        } else {
+            ThemeSendControlMessage(_theme, ID_CUSTOM_ASSOCIATE_FILES_CHECKBOX, BM_SETCHECK, BST_UNCHECKED, 0);
+            ThemeControlEnable(_theme, ID_CUSTOM_ASSOCIATE_FILES_CHECKBOX, FALSE);
+        }
+
+        LPWSTR targetDir = nullptr;
+        hr = BalGetStringVariable(L"TargetDir", &targetDir);
+        if (SUCCEEDED(hr) && targetDir && targetDir[0]) {
+            ThemeSetTextControl(_theme, ID_TARGETDIR_EDITBOX, targetDir);
+            StrFree(targetDir);
+        } else if (SUCCEEDED(hr)) {
+            StrFree(targetDir);
+            targetDir = nullptr;
+
+            LPWSTR defaultTargetDir = nullptr;
+            hr = BalGetStringVariable(L"DefaultCustomTargetDir", &defaultTargetDir);
+            if (SUCCEEDED(hr) && defaultTargetDir && !defaultTargetDir[0]) {
+                StrFree(defaultTargetDir);
+                defaultTargetDir = nullptr;
+                
+                hr = BalGetStringVariable(
+                    installAll ? L"DefaultAllUsersTargetDir" : L"DefaultJustForMeTargetDir",
+                    &defaultTargetDir
+                );
+            }
+            if (SUCCEEDED(hr) && defaultTargetDir) {
+                if (defaultTargetDir[0] && SUCCEEDED(BalFormatString(defaultTargetDir, &targetDir))) {
+                    ThemeSetTextControl(_theme, ID_TARGETDIR_EDITBOX, targetDir);
+                    StrFree(targetDir);
+                }
+                StrFree(defaultTargetDir);
+            }
+        }
+    }
+
+    void ModifyPage_Show() {
+        ThemeControlEnable(_theme, ID_REPAIR_BUTTON, !_suppressRepair);
+    }
+
+    void SuccessPage_Show() {
+        // on the "Success" page, check if the restart button should be enabled.
+        BOOL showRestartButton = FALSE;
+        LOC_STRING *successText = nullptr;
+        HRESULT hr = S_OK;
+        
+        if (_restartRequired) {
+            if (BOOTSTRAPPER_RESTART_PROMPT == _command.restart) {
+                showRestartButton = TRUE;
+            }
+        }
+
+        switch (_plannedAction) {
+        case BOOTSTRAPPER_ACTION_INSTALL:
+            hr = LocGetString(_wixLoc, L"#(loc.SuccessInstallMessage)", &successText);
+            break;
+        case BOOTSTRAPPER_ACTION_MODIFY:
+            hr = LocGetString(_wixLoc, L"#(loc.SuccessModifyMessage)", &successText);
+            break;
+        case BOOTSTRAPPER_ACTION_REPAIR:
+            hr = LocGetString(_wixLoc, L"#(loc.SuccessRepairMessage)", &successText);
+            break;
+        case BOOTSTRAPPER_ACTION_UNINSTALL:
+            hr = LocGetString(_wixLoc, L"#(loc.SuccessRemoveMessage)", &successText);
+            break;
+        }
+
+        if (successText) {
+            LPWSTR formattedString = nullptr;
+            BalFormatString(successText->wzText, &formattedString);
+            if (formattedString) {
+                ThemeSetTextControl(_theme, ID_SUCCESS_TEXT, formattedString);
+                StrFree(formattedString);
+            }
+        }
+
+        ThemeControlEnable(_theme, ID_SUCCESS_RESTART_TEXT, showRestartButton);
+        ThemeControlEnable(_theme, ID_SUCCESS_RESTART_BUTTON, showRestartButton);
+
+        if (_command.action != BOOTSTRAPPER_ACTION_INSTALL ||
+            !IsWindowsVersionOrGreater(10, 0, 0)) {
+            ThemeControlEnable(_theme, ID_SUCCESS_MAX_PATH_BUTTON, FALSE);
+        } else {
+            DWORD dataType = 0, buffer = 0, bufferLen = sizeof(buffer);
+            HKEY hKey;
+            LRESULT res = RegOpenKeyExW(
+                HKEY_LOCAL_MACHINE,
+                L"SYSTEM\\CurrentControlSet\\Control\\FileSystem",
+                0,
+                KEY_READ,
+                &hKey
+            );
+            if (res == ERROR_SUCCESS) {
+                res = RegQueryValueExW(hKey, L"LongPathsEnabled", nullptr, &dataType,
+                    (LPBYTE)&buffer, &bufferLen);
+                RegCloseKey(hKey);
+            }
+            else {
+                BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Failed to open SYSTEM\\CurrentControlSet\\Control\\FileSystem: error code %d", res);
+            }
+            if (res == ERROR_SUCCESS && dataType == REG_DWORD && buffer == 0) {
+                ThemeControlElevates(_theme, ID_SUCCESS_MAX_PATH_BUTTON, TRUE);
+            }
+            else {
+                if (res == ERROR_SUCCESS)
+                    BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Failed to read LongPathsEnabled value: error code %d", res);
+                else
+                    BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Hiding MAX_PATH button because it is already enabled");
+                ThemeControlEnable(_theme, ID_SUCCESS_MAX_PATH_BUTTON, FALSE);
+            }
+        }
+    }
+
+    void FailurePage_Show() {
+        // on the "Failure" page, show error message and check if the restart button should be enabled.
+
+        // if there is a log file variable then we'll assume the log file exists.
+        BOOL showLogLink = (_bundle.sczLogVariable && *_bundle.sczLogVariable);
+        BOOL showErrorMessage = FALSE;
+        BOOL showRestartButton = FALSE;
+
+        if (FAILED(_hrFinal)) {
+            LPWSTR unformattedText = nullptr;
+            LPWSTR text = nullptr;
+
+            // If we know the failure message, use that.
+            if (_failedMessage && *_failedMessage) {
+                StrAllocString(&unformattedText, _failedMessage, 0);
+            } else {
+                // try to get the error message from the error code.
+                StrAllocFromError(&unformattedText, _hrFinal, nullptr);
+                if (!unformattedText || !*unformattedText) {
+                    StrAllocFromError(&unformattedText, E_FAIL, nullptr);
+                }
+            }
+
+            if (E_WIXSTDBA_CONDITION_FAILED == _hrFinal) {
+                if (unformattedText) {
+                    StrAllocString(&text, unformattedText, 0);
+                }
+            } else {
+                StrAllocFormatted(&text, L"0x%08x - %ls", _hrFinal, unformattedText);
+            }
+
+            if (text) {
+                ThemeSetTextControl(_theme, ID_FAILURE_MESSAGE_TEXT, text);
+                showErrorMessage = TRUE;
+            }
+
+            ReleaseStr(text);
+            ReleaseStr(unformattedText);
+        }
+
+        if (_restartRequired && BOOTSTRAPPER_RESTART_PROMPT == _command.restart) {
+            showRestartButton = TRUE;
+        }
+
+        ThemeControlEnable(_theme, ID_FAILURE_LOGFILE_LINK, showLogLink);
+        ThemeControlEnable(_theme, ID_FAILURE_MESSAGE_TEXT, showErrorMessage);
+        ThemeControlEnable(_theme, ID_FAILURE_RESTART_TEXT, showRestartButton);
+        ThemeControlEnable(_theme, ID_FAILURE_RESTART_BUTTON, showRestartButton);
+    }
+
+    static void EnableMaxPathSupport() {
+        LPWSTR targetDir = nullptr, defaultDir = nullptr;
+        HRESULT hr = BalGetStringVariable(L"TargetDir", &targetDir);
+        if (FAILED(hr) || !targetDir || !targetDir[0]) {
+            BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Failed to get TargetDir");
+            return;
+        }
+
+        LPWSTR pythonw = nullptr;
+        StrAllocFormatted(&pythonw, L"%ls\\pythonw.exe", targetDir);
+        if (!pythonw || !pythonw[0]) {
+            BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Failed to construct pythonw.exe path");
+            return;
+        }
+
+        LPCWSTR arguments = L"-c \"import winreg; "
+            "winreg.SetValueEx("
+                "winreg.CreateKey(winreg.HKEY_LOCAL_MACHINE, "
+                    "r'SYSTEM\\CurrentControlSet\\Control\\FileSystem'), "
+                "'LongPathsEnabled', "
+                "None, "
+                "winreg.REG_DWORD, "
+                "1"
+            ")\"";
+        BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Executing %ls %ls", pythonw, arguments);
+        HINSTANCE res = ShellExecuteW(0, L"runas", pythonw, arguments, NULL, SW_HIDE);
+        BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "return code 0x%08x", res);
+    }
+
+public: // IBootstrapperApplication
+    virtual STDMETHODIMP OnStartup() {
+        HRESULT hr = S_OK;
+        DWORD dwUIThreadId = 0;
+
+        // create UI thread
+        _hUiThread = ::CreateThread(nullptr, 0, UiThreadProc, this, 0, &dwUIThreadId);
+        if (!_hUiThread) {
+            ExitWithLastError(hr, "Failed to create UI thread.");
+        }
+
+    LExit:
+        return hr;
+    }
+
+
+    virtual STDMETHODIMP_(int) OnShutdown() {
+        int nResult = IDNOACTION;
+
+        // wait for UI thread to terminate
+        if (_hUiThread) {
+            ::WaitForSingleObject(_hUiThread, INFINITE);
+            ReleaseHandle(_hUiThread);
+        }
+
+        // If a restart was required.
+        if (_restartRequired && _allowRestart) {
+            nResult = IDRESTART;
+        }
+
+        return nResult;
+    }
+
+    virtual STDMETHODIMP_(int) OnDetectRelatedMsiPackage(
+        __in_z LPCWSTR wzPackageId,
+        __in_z LPCWSTR /*wzProductCode*/,
+        __in BOOL fPerMachine,
+        __in DWORD64 /*dw64Version*/,
+        __in BOOTSTRAPPER_RELATED_OPERATION operation
+    ) {
+        if (BOOTSTRAPPER_RELATED_OPERATION_MAJOR_UPGRADE == operation && 
+            (CSTR_EQUAL == ::CompareStringW(LOCALE_NEUTRAL, 0, wzPackageId, -1, L"launcher_AllUsers", -1) ||
+             CSTR_EQUAL == ::CompareStringW(LOCALE_NEUTRAL, 0, wzPackageId, -1, L"launcher_JustForMe", -1))) {
+            auto hr = LoadAssociateFilesStateFromKey(_engine, fPerMachine ? HKEY_LOCAL_MACHINE : HKEY_CURRENT_USER);
+            if (hr == S_OK) {
+                _engine->SetVariableNumeric(L"AssociateFiles", 1);
+            } else if (FAILED(hr)) {
+                BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Failed to load AssociateFiles state: error code 0x%08X", hr);
+            }
+
+            _engine->SetVariableNumeric(L"Include_launcher", 1);
+            _engine->SetVariableNumeric(L"DetectedOldLauncher", 1);
+            _engine->SetVariableNumeric(L"InstallLauncherAllUsers", fPerMachine ? 1 : 0);
+        }
+        return CheckCanceled() ? IDCANCEL : IDNOACTION;
+    }
+
+    virtual STDMETHODIMP_(int) OnDetectRelatedBundle(
+        __in LPCWSTR wzBundleId,
+        __in BOOTSTRAPPER_RELATION_TYPE relationType,
+        __in LPCWSTR /*wzBundleTag*/,
+        __in BOOL fPerMachine,
+        __in DWORD64 /*dw64Version*/,
+        __in BOOTSTRAPPER_RELATED_OPERATION operation
+    ) {
+        BalInfoAddRelatedBundleAsPackage(&_bundle.packages, wzBundleId, relationType, fPerMachine);
+
+        // Remember when our bundle would cause a downgrade.
+        if (BOOTSTRAPPER_RELATED_OPERATION_DOWNGRADE == operation) {
+            _downgradingOtherVersion = TRUE;
+        } else if (BOOTSTRAPPER_RELATED_OPERATION_MAJOR_UPGRADE == operation) {
+            BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Detected previous version - planning upgrade");
+            _upgrading = TRUE;
+
+            LoadOptionalFeatureStates(_engine);
+        } else if (BOOTSTRAPPER_RELATED_OPERATION_NONE == operation) {
+            if (_command.action == BOOTSTRAPPER_ACTION_INSTALL) {
+                LOC_STRING *pLocString = nullptr;
+                if (SUCCEEDED(LocGetString(_wixLoc, L"#(loc.FailureExistingInstall)", &pLocString)) && pLocString) {
+                    BalFormatString(pLocString->wzText, &_failedMessage);
+                } else {
+                    BalFormatString(L"Cannot install [WixBundleName] because it is already installed.", &_failedMessage);
+                }
+                BalLog(
+                    BOOTSTRAPPER_LOG_LEVEL_ERROR,
+                    "Related bundle %ls is preventing install",
+                    wzBundleId
+                );
+                SetState(PYBA_STATE_FAILED, E_WIXSTDBA_CONDITION_FAILED);
+            }
+        }
+
+        return CheckCanceled() ? IDCANCEL : IDOK;
+    }
+
+
+    virtual STDMETHODIMP_(void) OnDetectPackageComplete(
+        __in LPCWSTR wzPackageId,
+        __in HRESULT hrStatus,
+        __in BOOTSTRAPPER_PACKAGE_STATE state
+    ) {
+        if (FAILED(hrStatus)) {
+            return;
+        }
+
+        BOOL detectedLauncher = FALSE;
+        HKEY hkey = HKEY_LOCAL_MACHINE;
+        if (CSTR_EQUAL == ::CompareStringW(LOCALE_NEUTRAL, 0, wzPackageId, -1, L"launcher_AllUsers", -1)) {
+            if (BOOTSTRAPPER_PACKAGE_STATE_PRESENT == state || BOOTSTRAPPER_PACKAGE_STATE_OBSOLETE == state) {
+                detectedLauncher = TRUE;
+                _engine->SetVariableNumeric(L"InstallLauncherAllUsers", 1);
+            }
+        } else if (CSTR_EQUAL == ::CompareStringW(LOCALE_NEUTRAL, 0, wzPackageId, -1, L"launcher_JustForMe", -1)) {
+            if (BOOTSTRAPPER_PACKAGE_STATE_PRESENT == state || BOOTSTRAPPER_PACKAGE_STATE_OBSOLETE == state) {
+                detectedLauncher = TRUE;
+                _engine->SetVariableNumeric(L"InstallLauncherAllUsers", 0);
+            }
+        }
+
+        if (detectedLauncher) {
+            /* When we detect the current version of the launcher. */
+            _engine->SetVariableNumeric(L"Include_launcher", 1);
+            _engine->SetVariableNumeric(L"DetectedLauncher", 1);
+            _engine->SetVariableString(L"Include_launcherState", L"disable");
+            _engine->SetVariableString(L"InstallLauncherAllUsersState", L"disable");
+
+            auto hr = LoadAssociateFilesStateFromKey(_engine, hkey);
+            if (hr == S_OK) {
+                _engine->SetVariableNumeric(L"AssociateFiles", 1);
+            } else if (FAILED(hr)) {
+                BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Failed to load AssociateFiles state: error code 0x%08X", hr);
+            }
+        }
+    }
+
+
+    virtual STDMETHODIMP_(void) OnDetectComplete(__in HRESULT hrStatus) {
+        if (SUCCEEDED(hrStatus) && _baFunction) {
+            BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Running detect complete BA function");
+            _baFunction->OnDetectComplete();
+        }
+
+        if (SUCCEEDED(hrStatus)) {
+            hrStatus = EvaluateConditions();
+        }
+
+        if (SUCCEEDED(hrStatus)) {
+            // Ensure the default path has been set
+            hrStatus = EnsureTargetDir();
+        }
+
+        SetState(PYBA_STATE_DETECTED, hrStatus);
+
+        // If we're not interacting with the user or we're doing a layout or we're just after a force restart
+        // then automatically start planning.
+        if (BOOTSTRAPPER_DISPLAY_FULL > _command.display ||
+            BOOTSTRAPPER_ACTION_LAYOUT == _command.action ||
+            BOOTSTRAPPER_ACTION_UNINSTALL == _command.action ||
+            BOOTSTRAPPER_RESUME_TYPE_REBOOT == _command.resumeType) {
+            if (SUCCEEDED(hrStatus)) {
+                ::PostMessageW(_hWnd, WM_PYBA_PLAN_PACKAGES, 0, _command.action);
+            }
+        }
+    }
+
+
+    virtual STDMETHODIMP_(int) OnPlanRelatedBundle(
+        __in_z LPCWSTR /*wzBundleId*/,
+        __inout_z BOOTSTRAPPER_REQUEST_STATE* pRequestedState
+    ) {
+        return CheckCanceled() ? IDCANCEL : IDOK;
+    }
+
+
+    virtual STDMETHODIMP_(int) OnPlanPackageBegin(
+        __in_z LPCWSTR wzPackageId,
+        __inout BOOTSTRAPPER_REQUEST_STATE *pRequestState
+    ) {
+        HRESULT hr = S_OK;
+        BAL_INFO_PACKAGE* pPackage = nullptr;
+
+        if (_nextPackageAfterRestart) {
+            // After restart we need to finish the dependency registration for our package so allow the package
+            // to go present.
+            if (CSTR_EQUAL == ::CompareStringW(LOCALE_NEUTRAL, 0, wzPackageId, -1, _nextPackageAfterRestart, -1)) {
+                // Do not allow a repair because that could put us in a perpetual restart loop.
+                if (BOOTSTRAPPER_REQUEST_STATE_REPAIR == *pRequestState) {
+                    *pRequestState = BOOTSTRAPPER_REQUEST_STATE_PRESENT;
+                }
+
+                ReleaseNullStr(_nextPackageAfterRestart); // no more skipping now.
+            } else {
+                // not the matching package, so skip it.
+                BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Skipping package: %ls, after restart because it was applied before the restart.", wzPackageId);
+
+                *pRequestState = BOOTSTRAPPER_REQUEST_STATE_NONE;
+            }
+        } else if ((_plannedAction == BOOTSTRAPPER_ACTION_INSTALL || _plannedAction == BOOTSTRAPPER_ACTION_MODIFY) &&
+                   SUCCEEDED(BalInfoFindPackageById(&_bundle.packages, wzPackageId, &pPackage))) {
+            BOOL f = FALSE;
+            if (SUCCEEDED(_engine->EvaluateCondition(pPackage->sczInstallCondition, &f)) && f) {
+                *pRequestState = BOOTSTRAPPER_REQUEST_STATE_PRESENT;
+            }
+        }
+
+        return CheckCanceled() ? IDCANCEL : IDOK;
+    }
+
+    virtual STDMETHODIMP_(int) OnPlanMsiFeature(
+        __in_z LPCWSTR wzPackageId,
+        __in_z LPCWSTR wzFeatureId,
+        __inout BOOTSTRAPPER_FEATURE_STATE* pRequestedState
+    ) {
+        LONGLONG install;
+        
+        if (wcscmp(wzFeatureId, L"AssociateFiles") == 0 || wcscmp(wzFeatureId, L"Shortcuts") == 0) {
+            if (SUCCEEDED(_engine->GetVariableNumeric(wzFeatureId, &install)) && install) {
+                *pRequestedState = BOOTSTRAPPER_FEATURE_STATE_LOCAL;
+            } else {
+                *pRequestedState = BOOTSTRAPPER_FEATURE_STATE_ABSENT;
+            }
+        } else {
+            *pRequestedState = BOOTSTRAPPER_FEATURE_STATE_LOCAL;
+        }
+        return CheckCanceled() ? IDCANCEL : IDNOACTION;
+    }
+
+    virtual STDMETHODIMP_(void) OnPlanComplete(__in HRESULT hrStatus) {
+        if (SUCCEEDED(hrStatus) && _baFunction) {
+            BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Running plan complete BA function");
+            _baFunction->OnPlanComplete();
+        }
+
+        SetState(PYBA_STATE_PLANNED, hrStatus);
+
+        if (SUCCEEDED(hrStatus)) {
+            ::PostMessageW(_hWnd, WM_PYBA_APPLY_PACKAGES, 0, 0);
+        }
+
+        _startedExecution = FALSE;
+        _calculatedCacheProgress = 0;
+        _calculatedExecuteProgress = 0;
+    }
+
+
+    virtual STDMETHODIMP_(int) OnCachePackageBegin(
+        __in_z LPCWSTR wzPackageId,
+        __in DWORD cCachePayloads,
+        __in DWORD64 dw64PackageCacheSize
+    ) {
+        if (wzPackageId && *wzPackageId) {
+            BAL_INFO_PACKAGE* pPackage = nullptr;
+            HRESULT hr = BalInfoFindPackageById(&_bundle.packages, wzPackageId, &pPackage);
+            LPCWSTR wz = (SUCCEEDED(hr) && pPackage->sczDisplayName) ? pPackage->sczDisplayName : wzPackageId;
+
+            ThemeSetTextControl(_theme, ID_CACHE_PROGRESS_PACKAGE_TEXT, wz);
+
+            // If something started executing, leave it in the overall progress text.
+            if (!_startedExecution) {
+                ThemeSetTextControl(_theme, ID_OVERALL_PROGRESS_PACKAGE_TEXT, wz);
+            }
+        }
+
+        return __super::OnCachePackageBegin(wzPackageId, cCachePayloads, dw64PackageCacheSize);
+    }
+
+
+    virtual STDMETHODIMP_(int) OnCacheAcquireProgress(
+        __in_z LPCWSTR wzPackageOrContainerId,
+        __in_z_opt LPCWSTR wzPayloadId,
+        __in DWORD64 dw64Progress,
+        __in DWORD64 dw64Total,
+        __in DWORD dwOverallPercentage
+    ) {
+        WCHAR wzProgress[5] = { };
+
+#ifdef DEBUG
+        BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "PYBA: OnCacheAcquireProgress() - container/package: %ls, payload: %ls, progress: %I64u, total: %I64u, overall progress: %u%%", wzPackageOrContainerId, wzPayloadId, dw64Progress, dw64Total, dwOverallPercentage);
+#endif
+
+        ::StringCchPrintfW(wzProgress, countof(wzProgress), L"%u%%", dwOverallPercentage);
+        ThemeSetTextControl(_theme, ID_CACHE_PROGRESS_TEXT, wzProgress);
+
+        ThemeSetProgressControl(_theme, ID_CACHE_PROGRESS_BAR, dwOverallPercentage);
+
+        _calculatedCacheProgress = dwOverallPercentage * PYBA_ACQUIRE_PERCENTAGE / 100;
+        ThemeSetProgressControl(_theme, ID_OVERALL_CALCULATED_PROGRESS_BAR, _calculatedCacheProgress + _calculatedExecuteProgress);
+
+        SetTaskbarButtonProgress(_calculatedCacheProgress + _calculatedExecuteProgress);
+
+        return __super::OnCacheAcquireProgress(wzPackageOrContainerId, wzPayloadId, dw64Progress, dw64Total, dwOverallPercentage);
+    }
+
+
+    virtual STDMETHODIMP_(int) OnCacheAcquireComplete(
+        __in_z LPCWSTR wzPackageOrContainerId,
+        __in_z_opt LPCWSTR wzPayloadId,
+        __in HRESULT hrStatus,
+        __in int nRecommendation
+    ) {
+        SetProgressState(hrStatus);
+        return __super::OnCacheAcquireComplete(wzPackageOrContainerId, wzPayloadId, hrStatus, nRecommendation);
+    }
+
+
+    virtual STDMETHODIMP_(int) OnCacheVerifyComplete(
+        __in_z LPCWSTR wzPackageId,
+        __in_z LPCWSTR wzPayloadId,
+        __in HRESULT hrStatus,
+        __in int nRecommendation
+    ) {
+        SetProgressState(hrStatus);
+        return __super::OnCacheVerifyComplete(wzPackageId, wzPayloadId, hrStatus, nRecommendation);
+    }
+
+
+    virtual STDMETHODIMP_(void) OnCacheComplete(__in HRESULT /*hrStatus*/) {
+        ThemeSetTextControl(_theme, ID_CACHE_PROGRESS_PACKAGE_TEXT, L"");
+        SetState(PYBA_STATE_CACHED, S_OK); // we always return success here and let OnApplyComplete() deal with the error.
+    }
+
+
+    virtual STDMETHODIMP_(int) OnError(
+        __in BOOTSTRAPPER_ERROR_TYPE errorType,
+        __in LPCWSTR wzPackageId,
+        __in DWORD dwCode,
+        __in_z LPCWSTR wzError,
+        __in DWORD dwUIHint,
+        __in DWORD /*cData*/,
+        __in_ecount_z_opt(cData) LPCWSTR* /*rgwzData*/,
+        __in int nRecommendation
+    ) {
+        int nResult = nRecommendation;
+        LPWSTR sczError = nullptr;
+
+        if (BOOTSTRAPPER_DISPLAY_EMBEDDED == _command.display) {
+            HRESULT hr = _engine->SendEmbeddedError(dwCode, wzError, dwUIHint, &nResult);
+            if (FAILED(hr)) {
+                nResult = IDERROR;
+            }
+        } else if (BOOTSTRAPPER_DISPLAY_FULL == _command.display) {
+            // If this is an authentication failure, let the engine try to handle it for us.
+            if (BOOTSTRAPPER_ERROR_TYPE_HTTP_AUTH_SERVER == errorType || BOOTSTRAPPER_ERROR_TYPE_HTTP_AUTH_PROXY == errorType) {
+                nResult = IDTRYAGAIN;
+            } else // show a generic error message box.
+            {
+                BalRetryErrorOccurred(wzPackageId, dwCode);
+
+                if (!_showingInternalUIThisPackage) {
+                    // If no error message was provided, use the error code to try and get an error message.
+                    if (!wzError || !*wzError || BOOTSTRAPPER_ERROR_TYPE_WINDOWS_INSTALLER != errorType) {
+                        HRESULT hr = StrAllocFromError(&sczError, dwCode, nullptr);
+                        if (FAILED(hr) || !sczError || !*sczError) {
+                            StrAllocFormatted(&sczError, L"0x%x", dwCode);
+                        }
+                    }
+
+                    nResult = ::MessageBoxW(_hWnd, sczError ? sczError : wzError, _theme->sczCaption, dwUIHint);
+                }
+            }
+
+            SetProgressState(HRESULT_FROM_WIN32(dwCode));
+        } else {
+            // just take note of the error code and let things continue.
+            BalRetryErrorOccurred(wzPackageId, dwCode);
+        }
+
+        ReleaseStr(sczError);
+        return nResult;
+    }
+
+
+    virtual STDMETHODIMP_(int) OnExecuteMsiMessage(
+        __in_z LPCWSTR wzPackageId,
+        __in INSTALLMESSAGE mt,
+        __in UINT uiFlags,
+        __in_z LPCWSTR wzMessage,
+        __in DWORD cData,
+        __in_ecount_z_opt(cData) LPCWSTR* rgwzData,
+        __in int nRecommendation
+    ) {
+#ifdef DEBUG
+        BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "PYBA: OnExecuteMsiMessage() - package: %ls, message: %ls", wzPackageId, wzMessage);
+#endif
+        if (BOOTSTRAPPER_DISPLAY_FULL == _command.display && (INSTALLMESSAGE_WARNING == mt || INSTALLMESSAGE_USER == mt)) {
+            int nResult = ::MessageBoxW(_hWnd, wzMessage, _theme->sczCaption, uiFlags);
+            return nResult;
+        }
+
+        if (INSTALLMESSAGE_ACTIONSTART == mt) {
+            ThemeSetTextControl(_theme, ID_EXECUTE_PROGRESS_ACTIONDATA_TEXT, wzMessage);
+        }

From pypy.commits at gmail.com  Mon Nov 11 08:35:28 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 11 Nov 2019 05:35:28 -0800 (PST)
Subject: [pypy-commit] pypy wininstaller_py2: adjust for python2
Message-ID: <5dc963a0.1c69fb81.e6e8a.d68a@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: wininstaller_py2
Changeset: r98025:8e4c447b8d6c
Date: 2019-11-11 08:22 -0500
http://bitbucket.org/pypy/pypy/changeset/8e4c447b8d6c/

Log:	adjust for python2

diff --git a/PCbuild/python.props b/PCbuild/python.props
--- a/PCbuild/python.props
+++ b/PCbuild/python.props
@@ -64,7 +64,7 @@
     <PyArchExt Condition="'$(ArchName)' == 'win32'">-32</PyArchExt>
     
     <!-- Full path of the resulting python.exe binary -->
-    <PythonExe Condition="'$(PythonExe)' == ''">$(InterpreterPath)pypy3-c$(PyDebugExt).exe</PythonExe>
+    <PythonExe Condition="'$(PythonExe)' == ''">$(InterpreterPath)pypy-c$(PyDebugExt).exe</PythonExe>
   </PropertyGroup>
   
   <PropertyGroup Condition="$(DefaultWindowsSDKVersion) == ''">
@@ -99,7 +99,7 @@
         ReleaseLevelNumber  -   10 for alpha, 11 for beta, 12 for RC (gamma), and 15 for final
         Field3Value         -   2101 for '3.5.2a1' (== 1000*2 + 10*10 ('a') + 1)
     -->
-    <_PatchLevelContent>$([System.IO.File]::ReadAllText(`$(PySourcePath)Include\patchlevel.h`))</_PatchLevelContent>
+    <_PatchLevelContent>$([System.IO.File]::ReadAllText(`$(PySourcePath)pypy\module\cpyext\include\patchlevel.h`))</_PatchLevelContent>
     <MajorVersionNumber>$([System.Text.RegularExpressions.Regex]::Match($(_PatchLevelContent), `define\s+PY_MAJOR_VERSION\s+(\d+)`).Groups[1].Value)</MajorVersionNumber>
     <MinorVersionNumber>$([System.Text.RegularExpressions.Regex]::Match($(_PatchLevelContent), `define\s+PY_MINOR_VERSION\s+(\d+)`).Groups[1].Value)</MinorVersionNumber>
     <MicroVersionNumber>$([System.Text.RegularExpressions.Regex]::Match($(_PatchLevelContent), `define\s+PY_MICRO_VERSION\s+(\d+)`).Groups[1].Value)</MicroVersionNumber>
diff --git a/PCbuild/setup.ico b/PCbuild/setup.ico
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..ab52102a8533bf956aadd8ae6bab13ecde1395b1
GIT binary patch

[cut]

diff --git a/PCbuild/win32/en-us/exe.msi b/PCbuild/win32/en-us/exe.msi
index a444b52839ec6e9ad8eadde9653e7dd370910671..83021f56d40b36fbebc3981b16204f8edca692ee
GIT binary patch

[cut]

diff --git a/pypy/tool/release/windowsinstaller/README b/pypy/tool/release/windowsinstaller/README
--- a/pypy/tool/release/windowsinstaller/README
+++ b/pypy/tool/release/windowsinstaller/README
@@ -9,7 +9,6 @@
     
 Steps:
     1. Translate the interpretter from the \pypy\goal folder.
-    2. Remove all __pycache__ directories (maybe do this in the script?)
-    3. Run buildrelease.bat from the visual studio command line.
+    2. Run buildrelease.bat from the top level directory.
 
-Now you should have .\PCbuild\win32\en-us\pypy-3.5.3.6609.exe
+Now you should have .\PCbuild\win32\en-us\pypy-*.exe
diff --git a/pypy/tool/release/windowsinstaller/bundle/bundle.wxs b/pypy/tool/release/windowsinstaller/bundle/bundle.wxs
--- a/pypy/tool/release/windowsinstaller/bundle/bundle.wxs
+++ b/pypy/tool/release/windowsinstaller/bundle/bundle.wxs
@@ -5,9 +5,9 @@
   <Bundle Name="!(loc.FullProductName)"
           UpgradeCode="$(var.CoreUpgradeCode)"
           Version="$(var.Version)"
-          IconSourceFile="..\..\..\PC\icons\setup.ico"
+          IconSourceFile="..\..\..\PCbuild\setup.ico"
           Manufacturer="!(loc.Manufacturer)"
-          AboutUrl="http://www.python.org/"
+          AboutUrl="http://www.pypy.org/"
           Compressed="no"
           dep:ProviderKey="CPython-$(var.MajorVersionNumber).$(var.MinorVersionNumber)$(var.PyArchExt)$(var.PyTestExt)">
     <BootstrapperApplication Id="PythonBA" SourceFile="$(var.BootstrapApp)">
diff --git a/pypy/tool/release/windowsinstaller/csv_to_wxs.py b/pypy/tool/release/windowsinstaller/csv_to_wxs.py
--- a/pypy/tool/release/windowsinstaller/csv_to_wxs.py
+++ b/pypy/tool/release/windowsinstaller/csv_to_wxs.py
@@ -21,8 +21,44 @@
 import sys
 
 from collections import defaultdict
-from itertools import chain, zip_longest
-from pathlib import PureWindowsPath
+try:
+    from itertools import chain, zip_longest
+    from pathlib import PureWindowsPath
+except ImportError:
+    #python2
+    from itertools import chain, izip_longest as zip_longest
+    import os
+    class PureWindowsPath(object):
+        def __init__(self, path):
+            self.path = path
+        @property
+        def suffix(self):
+            return os.path.splitext(self.path)[1]
+        @property
+        def parents(self):
+            p = self.path
+            ret = [self]
+            while True:
+                p_new = os.path.dirname(p)
+                if not p_new or p_new == p:
+                    break
+                ret.append(type(self)(p_new))
+                p = p_new
+            return ret[:-1]
+        @property
+        def parent(self):
+            return type(self)(os.path.dirname(self.path))
+        @property
+        def name(self):
+            return os.path.split(self.path)[1]
+        def __str__(self):
+            return self.path
+        def __hash__(self):
+            return hash(self.path)
+        def __eq__(self, other):
+            return self.path == str(other)
+
+    from io import open
 from uuid import uuid1
 
 ID_CHAR_SUBS = {
@@ -115,14 +151,21 @@
     # that we can skip rebuilding.
     try:
         with open(install_target, 'r') as f:
-            if all(x.rstrip('\r\n') == y for x, y in zip_longest(f, lines)):
-                print('File is up to date')
+            lasti = 0
+            for i, line in enumerate(f):
+                if i >=len(lines) or line != lines[i]:
+                    break
+                lasti = i
+            else:
+                if lasti == len(lines):
+                    print('File is up to date')
                 return
     except IOError:
         pass
 
     with open(install_target, 'w') as f:
-        f.writelines(line + '\n' for line in lines)
+        f.write(u'\n'.join(lines))
+        f.write(u'\n')
     print('Wrote {} lines to {}'.format(len(lines), install_target))
 
 if __name__ == '__main__':
diff --git a/pypy/tool/release/windowsinstaller/exe/exe.wxs b/pypy/tool/release/windowsinstaller/exe/exe.wxs
--- a/pypy/tool/release/windowsinstaller/exe/exe.wxs
+++ b/pypy/tool/release/windowsinstaller/exe/exe.wxs
@@ -19,8 +19,8 @@
         <Feature Id="Shortcuts" AllowAdvertise="no" Title="!(loc.Title)" Description="!(loc.Description)">
 			<ComponentGroupRef Id="exe_pypy" />
             <Component Id="exe_shortcut" Directory="MenuDir" Guid="*">
-                <Shortcut Id="pypy3_c"
-                          Target="[#pypy3_c]"
+                <Shortcut Id="pypy_c"
+                          Target="[#pypy_c]"
                           Name="!(loc.ShortcutName)"
                           Description="!(loc.ShortcutDescription)"
                           WorkingDirectory="InstallDirectory" />
diff --git a/pypy/tool/release/windowsinstaller/exe/exe_files.wxs b/pypy/tool/release/windowsinstaller/exe/exe_files.wxs
--- a/pypy/tool/release/windowsinstaller/exe/exe_files.wxs
+++ b/pypy/tool/release/windowsinstaller/exe/exe_files.wxs
@@ -15,20 +15,20 @@
         <PropertyRef Id="REGISTRYKEY" />
         <ComponentGroup Id="exe_pypy">
             <Component Id="exe_pypyc" Directory="InstallDirectory" Guid="$(var.PythonExeComponentGuid)">
-                <File Id="pypy3_c" Name="pypy3-c.exe" Source="!(bindpath.goal)pypy3-c.exe" KeyPath="yes" />
+                <File Id="pypy_c" Name="pypy-c.exe" Source="!(bindpath.goal)pypy-c.exe" KeyPath="yes" />
                 <RegistryKey Root="HKMU" Key="[REGISTRYKEY]">
                     <RegistryValue Key="InstallPath" Type="string" Value="[InstallDirectory]" KeyPath="no" />
-                    <RegistryValue Key="InstallPath" Name="ExecutablePath" Type="string" Value="[#pypy3_c]" KeyPath="no" />
+                    <RegistryValue Key="InstallPath" Name="ExecutablePath" Type="string" Value="[#pypy_c]" KeyPath="no" />
                 </RegistryKey>
             </Component>
             <Component Id="exe_pypycw" Directory="InstallDirectory" Guid="$(var.PythonwExeComponentGuid)">
-                <File Id="pypy3_cw" Name="pypy3-cw.exe" Source="!(bindpath.goal)pypy3-cw.exe" KeyPath="yes" />
+                <File Id="pypy_cw" Name="pypy-cw.exe" Source="!(bindpath.goal)pypy-cw.exe" KeyPath="yes" />
                 <RegistryKey Root="HKMU" Key="[REGISTRYKEY]">
-                    <RegistryValue Key="InstallPath" Name="WindowedExecutablePath" Type="string" Value="[#pypy3_cw]" KeyPath="no" />
+                    <RegistryValue Key="InstallPath" Name="WindowedExecutablePath" Type="string" Value="[#pypy_cw]" KeyPath="no" />
                 </RegistryKey>
             </Component>
-            <Component Id="libpypy3" Directory="InstallDirectory" Guid="*">
-                <File Name="libpypy3-c.dll" Source="!(bindpath.goal)libpypy3-c.dll" KeyPath="yes" />
+            <Component Id="libpypy" Directory="InstallDirectory" Guid="*">
+                <File Name="libpypy-c.dll" Source="!(bindpath.goal)libpypy-c.dll" KeyPath="yes" />
             </Component>
         </ComponentGroup>
     </Fragment>

From pypy.commits at gmail.com  Mon Nov 11 09:18:04 2019
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 11 Nov 2019 06:18:04 -0800 (PST)
Subject: [pypy-commit] pypy py3.6: Evaluate target only once when doing an
 annotated setitem
Message-ID: <5dc96d9c.1c69fb81.9e1d.cdb9@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6
Changeset: r98026:7d21b9cbf13a
Date: 2019-11-11 14:17 +0000
http://bitbucket.org/pypy/pypy/changeset/7d21b9cbf13a/

Log:	Evaluate target only once when doing an annotated setitem

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1008,11 +1008,12 @@
                 attr = target.value
                 self._annotation_evaluate(attr)
         elif isinstance(target, ast.Subscript):
-            # similar to the above, `a[0:5]: int` evaluates the name and the slice argument
-            # and if not in a function, also evaluates the annotation
-            sl = target.slice
-            self._annotation_evaluate(target.value)
-            self._annotation_eval_slice(sl)
+            if not assign.value:
+                # similar to the above, `a[0:5]: int` evaluates the name and the slice argument
+                # and if not in a function, also evaluates the annotation
+                sl = target.slice
+                self._annotation_evaluate(target.value)
+                self._annotation_eval_slice(sl)
         else:
             self.error("can't handle annotation with %s" % (target,), target)
         # if this is not in a function, evaluate the annotation
diff --git a/pypy/interpreter/test/apptest_annotations.py b/pypy/interpreter/test/apptest_annotations.py
--- a/pypy/interpreter/test/apptest_annotations.py
+++ b/pypy/interpreter/test/apptest_annotations.py
@@ -56,12 +56,8 @@
     C()
 
 def test_nonexistent_target():
-    try:
-        # this is invalid because `y` is undefined
-        # it should raise a NameError
+    with pytest.raises(NameError):
         y[0]: invalid
-    except NameError:
-        ...
 
 def test_non_simple_func_annotation():
     a = 5
@@ -136,3 +132,36 @@
         assert C.__annotations__ == {"cls": "abc"}
 
     f("abc")
+
+def test_side_effects():
+    calls = 0
+    def foo():
+        nonlocal calls
+        calls += 1
+    foo()
+    assert calls == 1
+    exec("foo()")
+    assert calls == 2
+    d = {}
+    exec("a: foo() = 1")
+    exec("b: foo()")
+    assert calls == 4
+    c = None
+    exec("c[foo()]: int")
+    assert calls == 5
+    def g():
+        c = None
+        c[foo()]: int
+    assert calls == 5
+    g()
+    assert calls == 6
+
+def test_side_effects_2():
+    calls = 0
+    def foo():
+        nonlocal calls
+        calls += 1
+    d = {}
+    exec("d[foo()]: int = 1")
+    assert d[None] == 1
+    assert calls == 1

From pypy.commits at gmail.com  Mon Nov 11 18:25:48 2019
From: pypy.commits at gmail.com (Yannick_Jadoul)
Date: Mon, 11 Nov 2019 15:25:48 -0800 (PST)
Subject: [pypy-commit] pypy py3.7-pep565: Implementing PEP 565,
 showing DeprecationWarning in __main__ by default
Message-ID: <5dc9edfc.1c69fb81.b223.c496@mx.google.com>

Author: Yannick Jadoul <yannick.jadoul at belgacom.net>
Branch: py3.7-pep565
Changeset: r98029:a21b521df584
Date: 2019-11-12 00:24 +0100
http://bitbucket.org/pypy/pypy/changeset/a21b521df584/

Log:	Implementing PEP 565, showing DeprecationWarning in __main__ by
	default

diff --git a/pypy/module/_warnings/interp_warnings.py b/pypy/module/_warnings/interp_warnings.py
--- a/pypy/module/_warnings/interp_warnings.py
+++ b/pypy/module/_warnings/interp_warnings.py
@@ -4,10 +4,11 @@
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
 from pypy.interpreter.error import OperationError, oefmt
 
-def create_filter(space, w_category, action):
+def create_filter(space, w_category, action, modname):
+    w_modname = space.newtext(modname) if modname is not None else space.w_None
     return space.newtuple([
         space.newtext(action), space.w_None, w_category,
-        space.w_None, space.newint(0)])
+        w_modname, space.newint(0)])
 
 class State:
     def __init__(self, space):
@@ -23,11 +24,13 @@
         filters_w = []
 
         filters_w.append(create_filter(
-            space, space.w_DeprecationWarning, "ignore"))
+            space, space.w_DeprecationWarning, "default", "__main__"))
         filters_w.append(create_filter(
-            space, space.w_PendingDeprecationWarning, "ignore"))
+            space, space.w_DeprecationWarning, "ignore", None))
         filters_w.append(create_filter(
-            space, space.w_ImportWarning, "ignore"))
+            space, space.w_PendingDeprecationWarning, "ignore", None))
+        filters_w.append(create_filter(
+            space, space.w_ImportWarning, "ignore", None))
 
         bytes_warning = space.sys.get_flag('bytes_warning')
         if bytes_warning > 1:
@@ -37,12 +40,12 @@
         else:
             action = "default"
         filters_w.append(create_filter(
-            space, space.w_BytesWarning, action))
+            space, space.w_BytesWarning, action, None))
 
         # note: in CPython, resource usage warnings are enabled by default
         # in pydebug mode
         filters_w.append(create_filter(
-            space, space.w_ResourceWarning, "ignore"))
+            space, space.w_ResourceWarning, "ignore", None))
 
         self.w_filters = space.newlist(filters_w)
 
@@ -165,8 +168,13 @@
     return (w_filename, lineno, w_module, w_registry)
 
 def check_matched(space, w_obj, w_arg):
+    # A 'None' filter always matches
     if space.is_w(w_obj, space.w_None):
         return True
+    # An internal plain text default filter must match exactly
+    if space.is_w(space.type(w_obj), space.w_unicode):
+        return space.eq_w(w_obj, w_arg)
+    # Otherwise assume a regex filter and call its match() method
     return space.is_true(space.call_method(w_obj, "match", w_arg))
 
 def get_filter(space, w_category, w_text, lineno, w_module):
diff --git a/pypy/module/_warnings/test/test_warnings.py b/pypy/module/_warnings/test/test_warnings.py
--- a/pypy/module/_warnings/test/test_warnings.py
+++ b/pypy/module/_warnings/test/test_warnings.py
@@ -5,7 +5,8 @@
         import _warnings
         assert _warnings._onceregistry == {}
         assert _warnings._defaultaction == 'default'
-        expected = [('ignore', None, DeprecationWarning, None, 0),
+        expected = [('default', None, DeprecationWarning, '__main__', 0),
+                    ('ignore', None, DeprecationWarning, None, 0),
                     ('ignore', None, PendingDeprecationWarning, None, 0),
                     ('ignore', None, ImportWarning, None, 0),
                     ('ignore', None, BytesWarning, None, 0),

From pypy.commits at gmail.com  Tue Nov 12 04:09:28 2019
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 12 Nov 2019 01:09:28 -0800 (PST)
Subject: [pypy-commit] pypy py3.7-pep565: Close branch py3.7-pep565
Message-ID: <5dca76c8.1c69fb81.57de6.2851@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.7-pep565
Changeset: r98030:0fbd282d801d
Date: 2019-11-12 09:08 +0000
http://bitbucket.org/pypy/pypy/changeset/0fbd282d801d/

Log:	Close branch py3.7-pep565


From pypy.commits at gmail.com  Tue Nov 12 04:09:50 2019
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 12 Nov 2019 01:09:50 -0800 (PST)
Subject: [pypy-commit] pypy py3.7: Merged in py3.7-pep565 (pull request #683)
Message-ID: <5dca76de.1c69fb81.7651b.09d2@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.7
Changeset: r98031:47c4c0ed3274
Date: 2019-11-12 09:08 +0000
http://bitbucket.org/pypy/pypy/changeset/47c4c0ed3274/

Log:	Merged in py3.7-pep565 (pull request #683)

	PEP 565: Show DeprecationWarning in __main__ by default

diff --git a/pypy/module/_warnings/interp_warnings.py b/pypy/module/_warnings/interp_warnings.py
--- a/pypy/module/_warnings/interp_warnings.py
+++ b/pypy/module/_warnings/interp_warnings.py
@@ -4,10 +4,11 @@
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
 from pypy.interpreter.error import OperationError, oefmt
 
-def create_filter(space, w_category, action):
+def create_filter(space, w_category, action, modname):
+    w_modname = space.newtext(modname) if modname is not None else space.w_None
     return space.newtuple([
         space.newtext(action), space.w_None, w_category,
-        space.w_None, space.newint(0)])
+        w_modname, space.newint(0)])
 
 class State:
     def __init__(self, space):
@@ -23,11 +24,13 @@
         filters_w = []
 
         filters_w.append(create_filter(
-            space, space.w_DeprecationWarning, "ignore"))
+            space, space.w_DeprecationWarning, "default", "__main__"))
         filters_w.append(create_filter(
-            space, space.w_PendingDeprecationWarning, "ignore"))
+            space, space.w_DeprecationWarning, "ignore", None))
         filters_w.append(create_filter(
-            space, space.w_ImportWarning, "ignore"))
+            space, space.w_PendingDeprecationWarning, "ignore", None))
+        filters_w.append(create_filter(
+            space, space.w_ImportWarning, "ignore", None))
 
         bytes_warning = space.sys.get_flag('bytes_warning')
         if bytes_warning > 1:
@@ -37,12 +40,12 @@
         else:
             action = "default"
         filters_w.append(create_filter(
-            space, space.w_BytesWarning, action))
+            space, space.w_BytesWarning, action, None))
 
         # note: in CPython, resource usage warnings are enabled by default
         # in pydebug mode
         filters_w.append(create_filter(
-            space, space.w_ResourceWarning, "ignore"))
+            space, space.w_ResourceWarning, "ignore", None))
 
         self.w_filters = space.newlist(filters_w)
 
@@ -165,8 +168,13 @@
     return (w_filename, lineno, w_module, w_registry)
 
 def check_matched(space, w_obj, w_arg):
+    # A 'None' filter always matches
     if space.is_w(w_obj, space.w_None):
         return True
+    # An internal plain text default filter must match exactly
+    if space.is_w(space.type(w_obj), space.w_unicode):
+        return space.eq_w(w_obj, w_arg)
+    # Otherwise assume a regex filter and call its match() method
     return space.is_true(space.call_method(w_obj, "match", w_arg))
 
 def get_filter(space, w_category, w_text, lineno, w_module):
diff --git a/pypy/module/_warnings/test/test_warnings.py b/pypy/module/_warnings/test/test_warnings.py
--- a/pypy/module/_warnings/test/test_warnings.py
+++ b/pypy/module/_warnings/test/test_warnings.py
@@ -5,7 +5,8 @@
         import _warnings
         assert _warnings._onceregistry == {}
         assert _warnings._defaultaction == 'default'
-        expected = [('ignore', None, DeprecationWarning, None, 0),
+        expected = [('default', None, DeprecationWarning, '__main__', 0),
+                    ('ignore', None, DeprecationWarning, None, 0),
                     ('ignore', None, PendingDeprecationWarning, None, 0),
                     ('ignore', None, ImportWarning, None, 0),
                     ('ignore', None, BytesWarning, None, 0),

From pypy.commits at gmail.com  Tue Nov 12 06:52:03 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 12 Nov 2019 03:52:03 -0800 (PST)
Subject: [pypy-commit] pypy msvc-discovery: delay problematic import
Message-ID: <5dca9ce3.1c69fb81.f0a1a.ee45@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: msvc-discovery
Changeset: r98032:ad2012524c8e
Date: 2019-11-12 06:51 -0500
http://bitbucket.org/pypy/pypy/changeset/ad2012524c8e/

Log:	delay problematic import

diff --git a/rpython/tool/setuptools_msvc.py b/rpython/tool/setuptools_msvc.py
--- a/rpython/tool/setuptools_msvc.py
+++ b/rpython/tool/setuptools_msvc.py
@@ -32,7 +32,6 @@
 import platform
 import itertools
 import distutils.errors
-from setuptools.extern.packaging.version import LegacyVersion
 
 from setuptools.extern.six.moves import filterfalse
 
@@ -187,6 +186,7 @@
     (for Numpy < 1.11.2)
     """
     if "numpy.distutils" in sys.modules:
+        from setuptools.extern.packaging.version import LegacyVersion
         import numpy as np
         if LegacyVersion(np.__version__) < LegacyVersion('1.11.2'):
             return np.distutils.ccompiler.gen_lib_options(*args, **kwargs)

From pypy.commits at gmail.com  Tue Nov 12 07:08:10 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 12 Nov 2019 04:08:10 -0800 (PST)
Subject: [pypy-commit] pypy msvc-discovery: merge default into branch
Message-ID: <5dcaa0aa.1c69fb81.e212.db59@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: msvc-discovery
Changeset: r98033:f1ab7867eb85
Date: 2019-11-12 06:58 -0500
http://bitbucket.org/pypy/pypy/changeset/f1ab7867eb85/

Log:	merge default into branch

diff --git a/rpython/rlib/compilerinfo.py b/rpython/rlib/compilerinfo.py
--- a/rpython/rlib/compilerinfo.py
+++ b/rpython/rlib/compilerinfo.py
@@ -18,8 +18,8 @@
 
 
 if platform.name == 'msvc':
-    # XXX hard-code the MSC version, I don't feel like computing it dynamically
-    _C_COMPILER_INFO = '"MSC v." Py_STR(_MSC_VER)'
+    # XXX hard-code the bit name
+    _C_COMPILER_INFO = '"MSC v." Py_STR(_MSC_VER) " 32 bit"'
 else:
     _C_COMPILER_INFO = '("GCC " __VERSION__)'
 

From pypy.commits at gmail.com  Tue Nov 12 07:08:12 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 12 Nov 2019 04:08:12 -0800 (PST)
Subject: [pypy-commit] pypy msvc-discovery: alsways prefer latest version of
 MSVC
Message-ID: <5dcaa0ac.1c69fb81.175c.ea27@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: msvc-discovery
Changeset: r98034:b00dc8eb081a
Date: 2019-11-12 07:06 -0500
http://bitbucket.org/pypy/pypy/changeset/b00dc8eb081a/

Log:	alsways prefer latest version of MSVC

diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py
--- a/rpython/translator/platform/windows.py
+++ b/rpython/translator/platform/windows.py
@@ -65,10 +65,8 @@
         log.error('looking for compiler %s raised exception "%s' % (vcver, str(e)))
         return None
 
-def find_msvc_env(x64flag=False, ver0=None):
+def find_msvc_env(x64flag=False):
     vcvers = [150, 140, 141, 90, 100]
-    if ver0 in vcvers:
-        vcvers.insert(0, ver0)
     for vsver in vcvers:
         env = _get_msvc_env(vsver, x64flag)
         if env is not None:
@@ -124,10 +122,7 @@
         patch_os_env(self.externals)
         self.c_environ = os.environ.copy()
         if cc is None:
-            # prefer compiler used to build host. Python2 only
-            if ver0 is None:
-                ver0 = _get_vcver0()
-            msvc_compiler_environ, self.vsver = find_msvc_env(x64, ver0=ver0)
+            msvc_compiler_environ, self.vsver = find_msvc_env(x64)
             Platform.__init__(self, 'cl.exe')
             if msvc_compiler_environ:
                 self.c_environ.update(msvc_compiler_environ)

From pypy.commits at gmail.com  Tue Nov 12 10:23:08 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 12 Nov 2019 07:23:08 -0800 (PST)
Subject: [pypy-commit] pypy code_page-utf8: fix translation
Message-ID: <5dcace5c.1c69fb81.e2d9f.eacb@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: code_page-utf8
Changeset: r98035:e1a3ccf336d0
Date: 2019-11-12 10:21 -0500
http://bitbucket.org/pypy/pypy/changeset/e1a3ccf336d0/

Log:	fix translation

diff --git a/pypy/interpreter/unicodehelper_win32.py b/pypy/interpreter/unicodehelper_win32.py
--- a/pypy/interpreter/unicodehelper_win32.py
+++ b/pypy/interpreter/unicodehelper_win32.py
@@ -1,8 +1,8 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib.runicode import (BOOLP, WideCharToMultiByte,
          MultiByteToWideChar)
-from rpython.rlib.rutf8 import (Utf8StringBuilder, Utf8StringIterator,
-         next_codepoint_pos)
+from rpython.rlib.rutf8 import (Utf8StringIterator, next_codepoint_pos,
+                                StringBuilder)
 from rpython.rlib import rwin32
 
 def Py_UNICODE_HIGH_SURROGATE(ch):
@@ -113,7 +113,7 @@
     flags = _decode_code_page_flags(cp)
     encoding = _code_page_name(cp)
     assert errorhandler is not None
-    res = Utf8StringBuilder(insize)
+    res = StringBuilder(insize)
     if errors == 'strict':
         _decode_helper(cp, s, flags, encoding, errors, errorhandler,
                        0, len(s), res)
@@ -152,7 +152,7 @@
     # Encode one codpoint at a time to allow the errorhandlers to do
     # their thing
     chars = lltype.malloc(rffi.CWCHARP.TO, 2, flavor = 'raw')
-    res = Utf8StringBuilder(lgt)
+    res = StringBuilder(lgt)
     try:
         pos = 0
         for uni in Utf8StringIterator(s):
@@ -187,7 +187,9 @@
                                            s, pos, pos + 1)
                         res.append(r)
                 else:
-                    res.append(buf.str(outsize))
+                    result = buf.str(outsize)
+                    assert result is not None
+                    res.append(result)
             pos += 1
         return res.build()
     finally:

From pypy.commits at gmail.com  Tue Nov 12 10:57:27 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 12 Nov 2019 07:57:27 -0800 (PST)
Subject: [pypy-commit] pypy py3.6: adapt sysconfig.py so pypy on windows
 adjusts the wheel ABI tag name
Message-ID: <5dcad667.1c69fb81.3141e.70dc@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r98036:24c4196d86af
Date: 2019-11-12 10:54 -0500
http://bitbucket.org/pypy/pypy/changeset/24c4196d86af/

Log:	adapt sysconfig.py so pypy on windows adjusts the wheel ABI tag name

diff --git a/lib-python/3/sysconfig.py b/lib-python/3/sysconfig.py
--- a/lib-python/3/sysconfig.py
+++ b/lib-python/3/sysconfig.py
@@ -450,6 +450,10 @@
     vars['EXE'] = '.exe'
     vars['VERSION'] = _PY_VERSION_SHORT_NO_DOT
     vars['BINDIR'] = os.path.dirname(_safe_realpath(sys.executable))
+    # pypy: give us control over the ABI tag in a wheel name
+    import _imp
+    so_ext = _imp.extension_suffixes()[0]
+    vars['SOABI']= '-'.join(so_ext.split('.')[1].split('-')[:2])
 
 #
 # public APIs

From pypy.commits at gmail.com  Tue Nov 12 22:35:54 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 12 Nov 2019 19:35:54 -0800 (PST)
Subject: [pypy-commit] pypy code_page-utf8: test, fix encoding code_pages
Message-ID: <5dcb7a1a.1c69fb81.6ab4c.5772@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: code_page-utf8
Changeset: r98040:5db4b2b481b5
Date: 2019-11-12 22:01 -0500
http://bitbucket.org/pypy/pypy/changeset/5db4b2b481b5/

Log:	test, fix encoding code_pages

diff --git a/pypy/interpreter/unicodehelper_win32.py b/pypy/interpreter/unicodehelper_win32.py
--- a/pypy/interpreter/unicodehelper_win32.py
+++ b/pypy/interpreter/unicodehelper_win32.py
@@ -163,12 +163,18 @@
                 charsize = 1
             else:
                 chars[0] = Py_UNICODE_HIGH_SURROGATE(uni)
-                chars[0] = Py_UNICODE_LOW_SURROGATE(uni)
+                chars[1] = Py_UNICODE_LOW_SURROGATE(uni)
                 charsize = 2
                 # first get the size of the result
             outsize = WideCharToMultiByte(cp, flags, chars, charsize, None, 0,
                                            None, used_default_p)
+            
             if outsize == 0:
+                if rwin32.GetLastError_saved() == rwin32.ERROR_NO_UNICODE_TRANSLATION:
+                    r, pos, retype = errorhandler(errors, name,
+                                       "invalid character", s, pos, pos+1)
+                    res.append(r)
+                    continue
                 raise rwin32.lastSavedWindowsError()
             # If we used a default char, then we failed!
             if (used_default_p and rffi.cast(lltype.Bool, used_default_p[0])):
diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -1,4 +1,5 @@
 import sys
+import pytest
 
 class AppTestCodecs:
     spaceconfig = {
@@ -525,6 +526,57 @@
             ):
             check_decode(1252, test)
 
+    def test_encode_65001(self):
+        tests = [
+            ('abc', 'strict', b'abc'),
+            ('\xe9\u20ac', 'strict',  b'\xc3\xa9\xe2\x82\xac'),
+            ('\U0010ffff', 'strict', b'\xf4\x8f\xbf\xbf'),
+            ('\udc80', 'strict', None),
+            ('\udc80', 'ignore', b''),
+            ('\udc80', 'replace', b'?'),
+            ('\udc80', 'backslashreplace', b'\\udc80'),
+            ('\udc80', 'namereplace', b'\\udc80'),
+            ('\udc80', 'surrogatepass', b'\xed\xb2\x80'),
+        ]
+        for text, errors, expected in tests:
+            if expected is not None:
+                try:
+                    encoded = text.encode('cp65001', errors)
+                except UnicodeEncodeError as err:
+                    assert False, ('Unable to encode %a to cp65001 with '
+                              'errors=%r: %s' % (text, errors, err))
+                assert encoded ==expected, ('%a.encode("cp65001", %r)=%a != %a'
+                    % (text, errors, encoded, expected))
+            else:
+                raises(UnicodeEncodeError, text.encode, "cp65001", errors)
+
+    def test_decode_65001(self):
+        tests = [
+            (b'abc', 'strict', 'abc'),
+            (b'\xc3\xa9\xe2\x82\xac', 'strict', '\xe9\u20ac'),
+            (b'\xf4\x8f\xbf\xbf', 'strict', '\U0010ffff'),
+            (b'\xef\xbf\xbd', 'strict', '\ufffd'),
+            (b'[\xc3\xa9]', 'strict', '[\xe9]'),
+            # invalid bytes
+            (b'[\xff]', 'strict', None),
+            (b'[\xff]', 'ignore', '[]'),
+            (b'[\xff]', 'replace', '[\ufffd]'),
+            (b'[\xff]', 'surrogateescape', '[\udcff]'),
+            (b'[\xed\xb2\x80]', 'strict', None),
+            (b'[\xed\xb2\x80]', 'ignore', '[]'),
+            (b'[\xed\xb2\x80]', 'replace', '[\ufffd\ufffd\ufffd]'),
+        ]
+        for raw, errors, expected in tests:
+            if expected is not None:
+                try:
+                    decoded = raw.decode('cp65001', errors)
+                except UnicodeDecodeError as err:
+                    assert False, ('Unable to decode %a from cp65001 with '
+                              'errors=%r: %s' % (raw, errors, err))
+                assert decoded == expected, ('%a.decode("cp65001", %r)=%a != %a'
+                    % (raw, errors, decoded, expected))
+            else:
+                raises(UnicodeDecodeError, raw.decode, 'cp65001', errors)
 
 
 class AppTestPartialEvaluation:

From pypy.commits at gmail.com  Tue Nov 12 22:35:56 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 12 Nov 2019 19:35:56 -0800 (PST)
Subject: [pypy-commit] pypy code_page-utf8: close branch to be merged
Message-ID: <5dcb7a1c.1c69fb81.98448.2a42@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: code_page-utf8
Changeset: r98041:3805e48e8f37
Date: 2019-11-12 22:31 -0500
http://bitbucket.org/pypy/pypy/changeset/3805e48e8f37/

Log:	close branch to be merged

diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -9,3 +9,11 @@
 .. branch: py3.6-asyncgen
 
 Fix asyncgen_hooks and refactor coroutine execution
+
+.. branch: py3.6-exc-info
+
+Follow CPython's use of exc_info more closely (issue 3096)
+
+.. branch: code_page-utf8
+
+Add encoding, decoding of codepages on windows

From pypy.commits at gmail.com  Tue Nov 12 22:35:57 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 12 Nov 2019 19:35:57 -0800 (PST)
Subject: [pypy-commit] pypy py3.6: merge code_page-utf8: adds encoding,
 decoding codepages on win32
Message-ID: <5dcb7a1d.1c69fb81.b3fba.49a7@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r98042:d3664d4fd243
Date: 2019-11-12 22:34 -0500
http://bitbucket.org/pypy/pypy/changeset/d3664d4fd243/

Log:	merge code_page-utf8: adds encoding, decoding codepages on win32

diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -9,3 +9,11 @@
 .. branch: py3.6-asyncgen
 
 Fix asyncgen_hooks and refactor coroutine execution
+
+.. branch: py3.6-exc-info
+
+Follow CPython's use of exc_info more closely (issue 3096)
+
+.. branch: code_page-utf8
+
+Add encoding, decoding of codepages on windows
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -61,11 +61,10 @@
     state = space.fromcache(interp_codecs.CodecState)
     errorhandler=state.decode_error_handler
     if _WIN32:
+        import pypy.interpreter.unicodehelper_win32 as win32
         bytes = space.bytes_w(w_string)
         slen = len(bytes)
-        uni, lgt = runicode.str_decode_mbcs(bytes, slen, 'strict', final=True,
-                           errorhandler=errorhandler, force_ignore=False)
-        utf8 = uni.encode('utf-8')
+        utf8, _, lgt = str_decode_mbcs(bytes, 'strict', True, errorhandler)
     elif 0 and  _MACOSX:
         bytes = space.bytes_w(w_string)
         utf8, lgt, pos  = str_decode_utf8(bytes, 'surrogateescape', True,
@@ -362,17 +361,31 @@
     return result.build()
 
 if _WIN32:
+    import pypy.interpreter.unicodehelper_win32 as win32
     def utf8_encode_mbcs(s, errors, errorhandler, allow_surrogates=False):
-        res = rutf8.utf8_encode_mbcs(s, errors, errorhandler,
-                                     force_replace=False)
+        res = win32.utf8_encode_mbcs(s, errors, errorhandler)
         return res
 
-    def str_decode_mbcs(s, errors, final, errorhandler, force_ignore=True):
-        slen = len(s)
-        res, size = runicode.str_decode_mbcs(s, slen, errors, final=final,
-                           errorhandler=errorhandler, force_ignore=force_ignore)
-        res_utf8 = runicode.unicode_encode_utf_8(res, size, 'strict')
-        return res_utf8, len(res), size
+    def str_decode_mbcs(s, errors, final, errorhandler):
+        res, size = win32.str_decode_mbcs(s, errors, errorhandler, final=final)
+        return res, len(res), size
+
+    def utf8_encode_oem(s, errors, errorhandler, allow_surrogates=False):
+        res = win32.utf8_encode_oem(s, errors, errorhandler)
+        return res
+
+    def str_decode_oem(s, errors, final, errorhandler):
+        res, size = win32.str_decode_oem(s, errors, errorhandler, final)
+        return res, len(res), size
+
+    def utf8_encode_code_page(cp, s, errors, errorhandler, allow_surrogates=False):
+        res = win32.utf8_encode_code_page(cp, s, errors, errorhandler)
+        return res
+
+    def str_decode_code_page(cp, s, errors, final, errorhandler):
+        res, size = win32.str_decode_code_page(cp, s, errors, errorhandler, final)
+        return res, len(res), size
+
 
 def str_decode_utf8(s, errors, final, errorhandler, allow_surrogates=False):
     try:
diff --git a/pypy/interpreter/unicodehelper_win32.py b/pypy/interpreter/unicodehelper_win32.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/unicodehelper_win32.py
@@ -0,0 +1,210 @@
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rlib.runicode import (BOOLP, WideCharToMultiByte,
+         MultiByteToWideChar)
+from rpython.rlib.rutf8 import (Utf8StringIterator, next_codepoint_pos,
+                                StringBuilder)
+from rpython.rlib import rwin32
+
+def Py_UNICODE_HIGH_SURROGATE(ch):
+   return rffi.cast(lltype.UniChar, 0xD800 - (0x10000 >> 10) + ((ch) >> 10))
+
+def Py_UNICODE_LOW_SURROGATE(ch):
+    return rffi.cast(lltype.UniChar, 0xDC00 + ((ch) & 0x3FF))
+
+if rffi.sizeof(rffi.INT) < rffi.sizeof(rffi.SIZE_T):
+    NEED_RETRY = True
+else:
+    NEED_RETRY = False
+WC_ERR_INVALID_CHARS = 0x0080
+
+code_page_map = {
+        rwin32.CP_ACP: "mbcs",
+        rwin32.CP_UTF7:"CP_UTF7",
+        rwin32.CP_UTF8:"CP_UTF8",
+    }
+
+def _code_page_name(code_page):
+    return code_page_map.get(code_page, "cp%d" % code_page)
+
+def _decode_code_page_flags(code_page):
+    if code_page == rwin32.CP_UTF7:
+        # The CP_UTF7 decoder only supports flags==0
+        return 0
+    return rwin32.MB_ERR_INVALID_CHARS
+
+def _encode_code_page_flags(code_page, errors):
+    if code_page == rwin32.CP_UTF8:
+        return WC_ERR_INVALID_CHARS
+    elif code_page == rwin32.CP_UTF7:
+        return 0
+    if errors == 'replace':
+        return 0
+    return rwin32.WC_NO_BEST_FIT_CHARS
+
+def _decode_cp_error(s, errorhandler, encoding, errors, start, end):
+    # late import to avoid circular import
+    from pypy.interpreter.unicodehelper import _str_decode_utf8_slowpath
+    if rwin32.GetLastError_saved() == rwin32.ERROR_NO_UNICODE_TRANSLATION:
+        msg = ("No mapping for the Unicode character exists in the target "
+               "multi-byte code page.")
+        r, ignore1, ignore2 = _str_decode_utf8_slowpath(s[start:end], errors, False, errorhandler, False)
+        return r, end
+    else:
+        raise rwin32.lastSavedWindowsError()
+
+def _unibuf_to_utf8(uni, insize):
+    """Encode the widechar unicode buffer u to utf8
+    Should never error, since the buffer comes from a call to
+    MultiByteToWideChar
+    """
+    flags = 0
+    cp = rwin32.CP_UTF8
+    used_default_p = lltype.nullptr(BOOLP.TO)
+    assert uni is not None
+    with rffi.scoped_nonmoving_unicodebuffer(uni) as dataptr:
+        # first get the size of the result
+        outsize = WideCharToMultiByte(cp, flags, dataptr, insize,
+                                    None, 0, None, used_default_p)
+        if outsize == 0:
+            raise rwin32.lastSavedWindowsError()
+        with rffi.scoped_alloc_buffer(outsize) as buf:
+            # do the conversion
+            if WideCharToMultiByte(cp, flags, dataptr, insize, buf.raw,
+                    outsize, None, used_default_p) == 0:
+                raise rwin32.lastSavedWindowsError()
+            result = buf.str(outsize)
+            assert result is not None
+            return result
+
+def _decode_helper(cp, s, flags, encoding, errors, errorhandler, 
+                   start, end, res):
+    if end > len(s):
+        end = len(s)
+    piece = s[start:end]
+    with rffi.scoped_nonmovingbuffer(piece) as dataptr:
+        # first get the size of the result
+        outsize = MultiByteToWideChar(cp, flags, dataptr, len(piece),
+                                    lltype.nullptr(rffi.CWCHARP.TO), 0)
+        if outsize == 0:
+            r, pos = _decode_cp_error(s, errorhandler,
+                                               encoding, errors, start, end)
+            res.append(r)
+            return pos
+
+        with rffi.scoped_alloc_unicodebuffer(outsize) as buf:
+            # do the conversion
+            if MultiByteToWideChar(cp, flags, dataptr, len(piece),
+                                   buf.raw, outsize) == 0:
+                r, pos = _decode_cp_error(s, errorhandler,
+                                             encoding, errors, start, end)
+                res.append(r)
+                return pos
+            else:
+                res.append(_unibuf_to_utf8(buf.str(outsize), outsize))
+    return end
+
+def str_decode_code_page(cp, s, errors, errorhandler, final=False):
+    """Decodes a byte string s from a code page cp with an error handler.
+    Returns utf8 result, original s length
+    """
+    insize = len(s)
+    if insize == 0:
+        return '', 0
+    flags = _decode_code_page_flags(cp)
+    encoding = _code_page_name(cp)
+    assert errorhandler is not None
+    res = StringBuilder(insize)
+    if errors == 'strict':
+        _decode_helper(cp, s, flags, encoding, errors, errorhandler,
+                       0, len(s), res)
+    else:
+        prev_pos = 0
+        pos = 0
+        while pos < len(s):
+            pos = next_codepoint_pos(s, prev_pos)
+            pos = _decode_helper(cp, s, flags, encoding,
+                            errors, errorhandler, prev_pos, pos, res)
+            prev_pos = pos
+    return res.build(), insize
+
+def str_decode_mbcs(s, errors, errorhandler, final=False):
+    return str_decode_code_page(rwin32.CP_ACP, s, errors, errorhandler, final)
+
+def str_decode_oem(s, errors, errorhandler, final=False):
+    return str_decode_code_page(rwin32.CP_OEMCP, s, errors, errorhandler, final)
+
+def utf8_encode_code_page(cp, s, errors, errorhandler):
+    """Encode a utf8 string s using code page cp and the given
+    errors/errorhandler.
+    Returns a encoded byte string
+    """
+
+    name = _code_page_name(cp)
+    lgt = len(s)
+
+    if lgt == 0:
+        return ''
+    flags = _encode_code_page_flags(cp, errors)
+    if cp in (rwin32.CP_UTF8, rwin32.CP_UTF7):
+        used_default_p = lltype.nullptr(BOOLP.TO)
+    else:
+        used_default_p = lltype.malloc(BOOLP.TO, 1, flavor='raw')
+    # Encode one codpoint at a time to allow the errorhandlers to do
+    # their thing
+    chars = lltype.malloc(rffi.CWCHARP.TO, 2, flavor = 'raw')
+    res = StringBuilder(lgt)
+    try:
+        pos = 0
+        for uni in Utf8StringIterator(s):
+            if used_default_p:
+                used_default_p[0] = rffi.cast(rwin32.BOOL, False)
+            if uni < 0x10000:
+                chars[0] = rffi.cast(lltype.UniChar, uni)
+                charsize = 1
+            else:
+                chars[0] = Py_UNICODE_HIGH_SURROGATE(uni)
+                chars[1] = Py_UNICODE_LOW_SURROGATE(uni)
+                charsize = 2
+                # first get the size of the result
+            outsize = WideCharToMultiByte(cp, flags, chars, charsize, None, 0,
+                                           None, used_default_p)
+            
+            if outsize == 0:
+                if rwin32.GetLastError_saved() == rwin32.ERROR_NO_UNICODE_TRANSLATION:
+                    r, pos, retype = errorhandler(errors, name,
+                                       "invalid character", s, pos, pos+1)
+                    res.append(r)
+                    continue
+                raise rwin32.lastSavedWindowsError()
+            # If we used a default char, then we failed!
+            if (used_default_p and rffi.cast(lltype.Bool, used_default_p[0])):
+                r, pos, retype = errorhandler(errors, name, "invalid character", s, pos, pos+1)
+                res.append(r)
+                continue
+            with rffi.scoped_alloc_buffer(outsize) as buf:
+                # do the conversion
+                if WideCharToMultiByte(cp, flags,
+                                       chars, charsize, buf.raw, outsize,
+                                       None, used_default_p) == 0:
+                    raise rwin32.lastSavedWindowsError()
+                if (used_default_p and
+                    rffi.cast(lltype.Bool, used_default_p[0])):
+                        r, pos, rettype = errorhandler(errors, name, "invalid character",
+                                           s, pos, pos + 1)
+                        res.append(r)
+                else:
+                    result = buf.str(outsize)
+                    assert result is not None
+                    res.append(result)
+            pos += 1
+        return res.build()
+    finally:
+        lltype.free(chars, flavor='raw')
+        if used_default_p:
+            lltype.free(used_default_p, flavor='raw')
+
+def utf8_encode_mbcs(s, errors, errorhandler):
+        return utf8_encode_code_page(rwin32.CP_ACP, s, errors, errorhandler)
+
+def utf8_encode_oem(s, errors, errorhandler):
+        return utf8_encode_code_page(rwin32.CP_OEMCP, s, errors, errorhandler)
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -3,7 +3,7 @@
 from rpython.rlib.objectmodel import we_are_translated, not_rpython
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 from rpython.rlib.rutf8 import MAXUNICODE
-from rpython.rlib.runicode import raw_unicode_escape_helper
+from rpython.rlib import runicode
 
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
@@ -291,7 +291,7 @@
         obj = w_obj._utf8
         while pos < end:
             code = rutf8.codepoint_at_pos(obj, pos)
-            raw_unicode_escape_helper(builder, code)
+            unicodehelper.raw_unicode_escape_helper(builder, code)
             pos = rutf8.next_codepoint_pos(obj, pos)
         return space.newtuple([space.newtext(builder.build()), w_end])
     elif space.isinstance_w(w_exc, space.w_UnicodeDecodeError):
@@ -303,7 +303,7 @@
         pos = start
         while pos < end:
             oc = ord(obj[pos])
-            raw_unicode_escape_helper(builder, oc)
+            unicodehelper.raw_unicode_escape_helper(builder, oc)
             pos += 1
         return space.newtuple([space.newtext(builder.build()), w_end])
     else:
@@ -705,10 +705,43 @@
          ]:
     make_decoder_wrapper(decoder)
 
-from rpython.rlib import runicode
-if hasattr(runicode, 'str_decode_mbcs'):
+if getattr(unicodehelper, '_WIN32', False):
     make_encoder_wrapper('mbcs_encode')
     make_decoder_wrapper('mbcs_decode')
+    make_encoder_wrapper('oem_encode')
+    make_decoder_wrapper('oem_decode')
+
+    # need to add the code_page argument
+
+    @unwrap_spec(code_page=int, errors='text_or_none')
+    def code_page_encode(space, code_page, w_arg, errors="strict"):
+        # w_arg is a W_Unicode or W_Bytes?
+        w_arg = space.convert_arg_to_w_unicode(w_arg, errors)
+        if errors is None:
+            errors = 'strict'
+        allow_surrogates = False
+        if errors in ('surrogatepass',):
+            allow_surrogates = True
+        state = space.fromcache(CodecState)
+        ulen = w_arg._length
+        result = unicodehelper.utf8_encode_code_page(code_page, w_arg._utf8,
+                      errors, state.encode_error_handler,
+                      allow_surrogates=allow_surrogates)
+        return space.newtuple([space.newbytes(result), space.newint(ulen)])
+
+    @unwrap_spec(code_page=int, string='bufferstr', errors='text_or_none',
+                 w_final=WrappedDefault(False))
+    def code_page_decode(space, code_page, string, errors="strict", w_final=None):
+        if errors is None:
+            errors = 'strict'
+        final = space.is_true(w_final)
+        state = space.fromcache(CodecState)
+        result, length, pos = unicodehelper.str_decode_code_page(code_page,
+                                   string, errors, final,
+                                   state.decode_error_handler)
+        # must return bytes, pos
+        return space.newtuple([space.newutf8(result, length), space.newint(pos)])
+
 
 # utf-8 functions are not regular, because we have to pass
 # "allow_surrogates=False"
diff --git a/pypy/module/_codecs/moduledef.py b/pypy/module/_codecs/moduledef.py
--- a/pypy/module/_codecs/moduledef.py
+++ b/pypy/module/_codecs/moduledef.py
@@ -1,5 +1,6 @@
 from pypy.interpreter.mixedmodule import MixedModule
 from rpython.rlib.objectmodel import not_rpython
+from rpython.rlib import rwin32
 from pypy.module._codecs import interp_codecs
 
 class Module(MixedModule):
@@ -87,11 +88,14 @@
 
     @not_rpython
     def __init__(self, space, *args):
-        # mbcs codec is Windows specific, and based on rffi.
-        from rpython.rlib import runicode
-        if (hasattr(runicode, 'str_decode_mbcs')):
+        # mbcs codec is Windows specific, and based on rffi system calls.
+        if rwin32.WIN32:
             self.interpleveldefs['mbcs_encode'] = 'interp_codecs.mbcs_encode'
+            self.interpleveldefs['oem_encode'] = 'interp_codecs.oem_encode'
+            self.interpleveldefs['code_page_encode'] = 'interp_codecs.code_page_encode'
             self.interpleveldefs['mbcs_decode'] = 'interp_codecs.mbcs_decode'
+            self.interpleveldefs['oem_decode'] = 'interp_codecs.oem_decode'
+            self.interpleveldefs['code_page_decode'] = 'interp_codecs.code_page_decode'
 
         MixedModule.__init__(self, space, *args)
 
diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -1,4 +1,5 @@
 import sys
+import pytest
 
 class AppTestCodecs:
     spaceconfig = {
@@ -432,6 +433,151 @@
             assert((b'aaaa' + seq + b'bbbb').decode('utf-8', 'ignore') ==
                           'aaaa' + res + 'bbbb')
 
+ at pytest.mark.skipif(sys.platform != 'win32', reason='win32-only')
+class AppTestCodePage:
+    spaceconfig = {
+    }
+
+    def test_code_pages(self):
+        import _codecs as codecs
+        def check_decode(cp, test):
+            raw, errors, expected  = test
+            if expected is not None:
+                try:
+                    decoded = codecs.code_page_decode(cp, raw, errors, True)
+                except UnicodeDecodeError as err:
+                    assert False, ('Unable to decode %a from "cp%s" with '
+                              'errors=%r: %s' % (raw, cp, errors, err))
+                assert decoded[0] == expected, ('%a.decode("cp%s", %r)=%a != %a'
+                    % (raw, cp, errors, decoded[0], expected))
+                assert decoded[1] >= 0
+                assert decoded[1] <= len(raw)
+            else:
+                raises(UnicodeDecodeError,
+                    codecs.code_page_decode, cp, raw, errors, True)
+
+        def check_encode(cp, test):
+            text, errors, expected = test
+            if expected is not None:
+                try:
+                    encoded = codecs.code_page_encode(cp, text, errors)
+                except UnicodeEncodeError as err:
+                    assert False, ('Unable to encode %a to "cp%s" with '
+                              'errors=%r: %s' % (text, cp, errors, err))
+                assert encoded[0] == expected, ('%a.encode("cp%s", %r)=%a != %a'
+                    % (text, cp, errors, encoded[0], expected))
+                assert encoded[1] == len(text)
+            else:
+                raises(UnicodeEncodeError,
+                    codecs.code_page_encode, cp, text, errors)
+
+        for test in (
+                (u'abc', 'strict', b'abc'),
+                (u'\uff44\u9a3e', 'strict', b'\x82\x84\xe9\x80'),
+                # test error handlers
+                (u'\xff', 'strict', None),
+                (u'[\xff]', 'ignore', b'[]'),
+                (u'[\xff]', 'replace', b'[y]'),
+                (u'[\u20ac]', 'replace', b'[?]'),
+                (u'[\xff]', 'backslashreplace', b'[\\xff]'),
+                (u'[\xff]', 'namereplace',
+                 b'[\\N{LATIN SMALL LETTER Y WITH DIAERESIS}]'),
+                (u'[\xff]', 'xmlcharrefreplace', b'[&#255;]'),
+                (u'\udcff', 'strict', None),
+                (u'[\udcff]', 'surrogateescape', b'[\xff]'),
+                (u'[\udcff]', 'surrogatepass', None),
+            ):
+            check_encode(932, test)
+
+        for test in (
+                (b'abc', 'strict', u'abc'),
+                (b'\x82\x84\xe9\x80', 'strict', u'\uff44\u9a3e'),
+                # invalid bytes
+                (b'[\xff]', 'strict', None),
+                (b'[\xff]', 'ignore', u'[]'),
+                (b'[\xff]', 'replace', u'[\ufffd]'),
+                (b'[\xff]', 'backslashreplace', u'[\\xff]'),
+                (b'[\xff]', 'surrogateescape', u'[\udcff]'),
+                (b'[\xff]', 'surrogatepass', None),
+                (b'\x81\x00abc', 'strict', None),
+                (b'\x81\x00abc', 'ignore', u'\x00abc'),
+                (b'\x81\x00abc', 'replace', u'\ufffd\x00abc'),
+                (b'\x81\x00abc', 'backslashreplace', u'\\x81\x00abc'),
+            ):
+            check_decode(932, test)
+
+        for test in (
+                (u'abc', 'strict', b'abc'),
+                (u'\xe9\u20ac', 'strict',  b'\xe9\x80'),
+                (u'\xff', 'strict', b'\xff'),
+                # test error handlers
+                (u'\u0141', 'strict', None),
+                (u'\u0141', 'ignore', b''),
+                (u'\u0141', 'replace', b'L'),
+                (u'\udc98', 'surrogateescape', b'\x98'),
+                (u'\udc98', 'surrogatepass', None),
+            ):
+            check_encode(1252, test)
+
+        for test in (
+                (b'abc', 'strict', u'abc'),
+                (b'\xe9\x80', 'strict', u'\xe9\u20ac'),
+                (b'\xff', 'strict', u'\xff'),
+            ):
+            check_decode(1252, test)
+
+    def test_encode_65001(self):
+        tests = [
+            ('abc', 'strict', b'abc'),
+            ('\xe9\u20ac', 'strict',  b'\xc3\xa9\xe2\x82\xac'),
+            ('\U0010ffff', 'strict', b'\xf4\x8f\xbf\xbf'),
+            ('\udc80', 'strict', None),
+            ('\udc80', 'ignore', b''),
+            ('\udc80', 'replace', b'?'),
+            ('\udc80', 'backslashreplace', b'\\udc80'),
+            ('\udc80', 'namereplace', b'\\udc80'),
+            ('\udc80', 'surrogatepass', b'\xed\xb2\x80'),
+        ]
+        for text, errors, expected in tests:
+            if expected is not None:
+                try:
+                    encoded = text.encode('cp65001', errors)
+                except UnicodeEncodeError as err:
+                    assert False, ('Unable to encode %a to cp65001 with '
+                              'errors=%r: %s' % (text, errors, err))
+                assert encoded ==expected, ('%a.encode("cp65001", %r)=%a != %a'
+                    % (text, errors, encoded, expected))
+            else:
+                raises(UnicodeEncodeError, text.encode, "cp65001", errors)
+
+    def test_decode_65001(self):
+        tests = [
+            (b'abc', 'strict', 'abc'),
+            (b'\xc3\xa9\xe2\x82\xac', 'strict', '\xe9\u20ac'),
+            (b'\xf4\x8f\xbf\xbf', 'strict', '\U0010ffff'),
+            (b'\xef\xbf\xbd', 'strict', '\ufffd'),
+            (b'[\xc3\xa9]', 'strict', '[\xe9]'),
+            # invalid bytes
+            (b'[\xff]', 'strict', None),
+            (b'[\xff]', 'ignore', '[]'),
+            (b'[\xff]', 'replace', '[\ufffd]'),
+            (b'[\xff]', 'surrogateescape', '[\udcff]'),
+            (b'[\xed\xb2\x80]', 'strict', None),
+            (b'[\xed\xb2\x80]', 'ignore', '[]'),
+            (b'[\xed\xb2\x80]', 'replace', '[\ufffd\ufffd\ufffd]'),
+        ]
+        for raw, errors, expected in tests:
+            if expected is not None:
+                try:
+                    decoded = raw.decode('cp65001', errors)
+                except UnicodeDecodeError as err:
+                    assert False, ('Unable to decode %a from cp65001 with '
+                              'errors=%r: %s' % (raw, errors, err))
+                assert decoded == expected, ('%a.decode("cp65001", %r)=%a != %a'
+                    % (raw, errors, decoded, expected))
+            else:
+                raises(UnicodeDecodeError, raw.decode, 'cp65001', errors)
+
 
 class AppTestPartialEvaluation:
     spaceconfig = dict(usemodules=['array',])
diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py
--- a/rpython/rlib/rwin32.py
+++ b/rpython/rlib/rwin32.py
@@ -114,6 +114,7 @@
                        WC_NO_BEST_FIT_CHARS STD_INPUT_HANDLE STD_OUTPUT_HANDLE
                        STD_ERROR_HANDLE HANDLE_FLAG_INHERIT FILE_TYPE_CHAR
                        LOAD_WITH_ALTERED_SEARCH_PATH
+                       CP_ACP CP_UTF8 CP_UTF7 CP_OEMCP MB_ERR_INVALID_CHARS
                     """
         from rpython.translator.platform import host_factory
         static_platform = host_factory()

From pypy.commits at gmail.com  Wed Nov 13 10:26:20 2019
From: pypy.commits at gmail.com (mattip)
Date: Wed, 13 Nov 2019 07:26:20 -0800 (PST)
Subject: [pypy-commit] pypy default: backport changes from py3.6 to minimize
 diff
Message-ID: <5dcc209c.1c69fb81.501f7.2c70@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r98043:65f80b74784f
Date: 2019-11-13 07:51 -0500
http://bitbucket.org/pypy/pypy/changeset/65f80b74784f/

Log:	backport changes from py3.6 to minimize diff

diff --git a/pypy/tool/build_cffi_imports.py b/pypy/tool/build_cffi_imports.py
--- a/pypy/tool/build_cffi_imports.py
+++ b/pypy/tool/build_cffi_imports.py
@@ -24,7 +24,19 @@
 # the OS, such as a recent openssl/libressl.
 curdir = os.path.abspath(os.path.dirname(__file__))
 deps_destdir = os.path.join(curdir, 'dest')
+configure_args = ['./configure',
+            '--prefix=/usr',
+            '--disable-shared',
+            '--enable-silent-rules',
+            '--disable-dependency-tracking',
+        ]
 cffi_dependencies = {
+    'lzma': ('https://tukaani.org/xz/xz-5.2.3.tar.gz',
+             '71928b357d0a09a12a4b4c5fafca8c31c19b0e7d3b8ebb19622e96f26dbf28cb',
+             [configure_args,
+              ['make', '-s', '-j', str(multiprocessing.cpu_count())],
+              ['make', 'install', 'DESTDIR={}/'.format(deps_destdir)],
+             ]),
     '_ssl': ('https://www.openssl.org/source/openssl-1.1.1c.tar.gz',
              'f6fb3079ad15076154eda9413fed42877d668e7069d9b87396d0804fdb3f4c90',
              [['./config', '--prefix=/usr', 'no-shared'],
@@ -33,7 +45,7 @@
              ]),
     '_gdbm': ('http://ftp.gnu.org/gnu/gdbm/gdbm-1.13.tar.gz',
               '9d252cbd7d793f7b12bcceaddda98d257c14f4d1890d851c386c37207000a253',
-             [['./config', '--without-readline'],
+              [configure_args,
               ['make', '-s', '-j', str(multiprocessing.cpu_count())],
               ['make', 'install', 'DESTDIR={}/'.format(deps_destdir)],
              ]),
@@ -176,8 +188,11 @@
             status, stdout, stderr = run_subprocess(str(pypy_c), args,
                                                     cwd=cwd, env=env)
             if status != 0:
-                print(stdout, stderr, file=sys.stderr)
                 failures.append((key, module))
+                print("stdout:")
+                print(stdout, file=sys.stderr)
+                print("stderr:")
+                print(stderr, file=sys.stderr)
         except:
             import traceback;traceback.print_exc()
             failures.append((key, module))
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -30,6 +30,8 @@
 
 STDLIB_VER = "2.7"
 
+POSIX_EXE = 'pypy'
+
 from pypy.tool.build_cffi_imports import (create_cffi_import_libraries,
         MissingDependenciesError, cffi_build_scripts)
 
@@ -71,7 +73,7 @@
 
     basedir = py.path.local(basedir)
     if not override_pypy_c:
-        basename = 'pypy-c'
+        basename = POSIX_EXE + '-c'
         if sys.platform == 'win32':
             basename += '.exe'
         pypy_c = basedir.join('pypy', 'goal', basename)
@@ -106,10 +108,13 @@
 
     if (sys.platform != 'win32' and    # handled below
         not _fake and os.path.getsize(str(pypy_c)) < 500000):
-        # This pypy-c is very small, so it means it relies on libpypy_c.so.
+        # This 'pypy_c' is very small, so it means it relies on a so/dll
         # If it would be bigger, it wouldn't.  That's a hack.
-        libpypy_name = ('libpypy-c.so' if not sys.platform.startswith('darwin')
-                                       else 'libpypy-c.dylib')
+        if sys.platform.startswith('darwin'):
+            ext = 'dylib'
+        else:
+            ext = 'so'
+        libpypy_name = 'lib' + POSIX_EXE + '-c.' + ext
         libpypy_c = pypy_c.new(basename=libpypy_name)
         if not libpypy_c.check():
             raise PyPyCNotFound('Expected pypy to be mostly in %r, but did '
@@ -134,8 +139,9 @@
             tgt = py.path.local(tgt)
             binaries.append((pypyw, tgt.new(purebasename=tgt.purebasename + 'w').basename, None))
             print("Picking %s" % str(pypyw))
-        # Can't rename a DLL: it is always called 'libpypy-c.dll'
-        win_extras = [('libpypy-c.dll', None), ('sqlite3.dll', lib_pypy)]
+        # Can't rename a DLL
+        win_extras = [('lib' + POSIX_EXE + '-c.dll', None),
+                      ('sqlite3.dll', lib_pypy)]
         if not options.no_tk:
             tkinter_dir = lib_pypy.join('_tkinter')
             win_extras += [('tcl85.dll', tkinter_dir), ('tk85.dll', tkinter_dir)]
@@ -155,9 +161,7 @@
             print('Picking %s (and contents)' % libsdir)
             shutil.copytree(str(libsdir), str(pypydir.join('libs')))
         else:
-            print('"libs" dir with import library not found.')
-            print('You have to create %r' % (str(libsdir),))
-            print('and copy libpypy-c.lib in there, renamed to python27.lib')
+            raise RuntimError('"libs" dir with import library not found.')
             # XXX users will complain that they cannot compile capi (cpyext)
             # modules for windows, also embedding pypy (i.e. in cffi)
             # will fail.
@@ -223,6 +227,14 @@
         else:
             open(str(archive), 'wb').close()
         os.chmod(str(archive), 0755)
+    #if not _fake and not sys.platform == 'win32':
+    #    # create the pypy3 symlink
+    #    old_dir = os.getcwd()
+    #    os.chdir(str(bindir))
+    #    try:
+    #        os.symlink(POSIX_EXE, 'pypy3')
+    #    finally:
+    #        os.chdir(old_dir)
     fix_permissions(pypydir)
 
     old_dir = os.getcwd()
@@ -292,9 +304,9 @@
             setattr(ns, self.dest, option[2:4] != 'no')
 
     if sys.platform == 'win32':
-        pypy_exe = 'pypy.exe'
+        pypy_exe = POSIX_EXE + '.exe'
     else:
-        pypy_exe = 'pypy'
+        pypy_exe = POSIX_EXE
     parser = argparse.ArgumentParser()
     args = list(args)
     if args:
diff --git a/pypy/tool/release/test/test_package.py b/pypy/tool/release/test/test_package.py
--- a/pypy/tool/release/test/test_package.py
+++ b/pypy/tool/release/test/test_package.py
@@ -1,4 +1,4 @@
-
+import os
 import py
 from pypy import pypydir
 from pypy.tool.release import package
@@ -15,8 +15,8 @@
             cls.exe_name_in_archive = 'pypy-c.exe'
         else:
             basename = 'pypy-c'
-            cls.rename_pypy_c = 'pypy'
-            cls.exe_name_in_archive = 'bin/pypy'
+            cls.rename_pypy_c = package.POSIX_EXE
+            cls.exe_name_in_archive = os.path.join('bin', package.POSIX_EXE)
         cls.pypy_c = py.path.local(pypydir).join('goal', basename)
 
     def test_dir_structure(self, test='test'):
diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py
--- a/rpython/rlib/rstring.py
+++ b/rpython/rlib/rstring.py
@@ -601,6 +601,11 @@
         assert i >= self.start
         self.i = i
         c = self.s[i]
+        if self.allow_underscores and c == '_':
+            i = self.i - 1
+            assert i >= 0
+            self.i = i
+            c = self.s[i]
         digit = ord(c)
         if '0' <= c <= '9':
             digit -= ord('0')
diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py
--- a/rpython/rlib/rwin32.py
+++ b/rpython/rlib/rwin32.py
@@ -114,6 +114,7 @@
                        WC_NO_BEST_FIT_CHARS STD_INPUT_HANDLE STD_OUTPUT_HANDLE
                        STD_ERROR_HANDLE HANDLE_FLAG_INHERIT FILE_TYPE_CHAR
                        LOAD_WITH_ALTERED_SEARCH_PATH
+                       CP_ACP CP_UTF8 CP_UTF7 CP_OEMCP MB_ERR_INVALID_CHARS
                     """
         from rpython.translator.platform import host_factory
         static_platform = host_factory()

From pypy.commits at gmail.com  Wed Nov 13 10:26:24 2019
From: pypy.commits at gmail.com (mattip)
Date: Wed, 13 Nov 2019 07:26:24 -0800 (PST)
Subject: [pypy-commit] pypy default: ignore cffi build artifacts that also
 happen to be over 260 chars in length
Message-ID: <5dcc20a0.1c69fb81.ba9be.f4fd@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r98044:874d998bc429
Date: 2019-11-13 10:19 -0500
http://bitbucket.org/pypy/pypy/changeset/874d998bc429/

Log:	ignore cffi build artifacts that also happen to be over 260 chars in
	length

diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -189,14 +189,14 @@
                           for source, target, target_dir in binaries])
 
     # Careful: to copy lib_pypy, copying just the hg-tracked files
-    # would not be enough: there are also ctypes_config_cache/_*_cache.py.
-    # XXX ^^^ this is no longer true!
+    # would not be enough: there are also build artifacts like cffi-generated
+    # dynamic libs
     shutil.copytree(str(basedir.join('lib-python').join(STDLIB_VER)),
                     str(pypydir.join('lib-python').join(STDLIB_VER)),
                     ignore=ignore_patterns('.svn', 'py', '*.pyc', '*~'))
     shutil.copytree(str(basedir.join('lib_pypy')), str(lib_pypy),
                     ignore=ignore_patterns('.svn', 'py', '*.pyc', '*~',
-                                           '*_cffi.c', '*.o'))
+                                           '*_cffi.c', '*.o', '*.pyd-*'))
     for file in ['README.rst',]:
         shutil.copy(str(basedir.join(file)), str(pypydir))
     for file in ['_testcapimodule.c', '_ctypes_test.c']:

From pypy.commits at gmail.com  Wed Nov 13 10:26:26 2019
From: pypy.commits at gmail.com (mattip)
Date: Wed, 13 Nov 2019 07:26:26 -0800 (PST)
Subject: [pypy-commit] pypy py3.6: merge default into py3.6
Message-ID: <5dcc20a2.1c69fb81.5ec18.c520@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r98045:7d4bfee3b8a6
Date: 2019-11-13 10:25 -0500
http://bitbucket.org/pypy/pypy/changeset/7d4bfee3b8a6/

Log:	merge default into py3.6

diff --git a/pypy/tool/build_cffi_imports.py b/pypy/tool/build_cffi_imports.py
--- a/pypy/tool/build_cffi_imports.py
+++ b/pypy/tool/build_cffi_imports.py
@@ -194,9 +194,9 @@
             if status != 0:
                 failures.append((key, module))
                 print("stdout:")
-                print(stdout.decode('utf-8'))
+                print(stdout.decode('utf-8'), file=sys.stderr)
                 print("stderr:")
-                print(stderr)
+                print(stderr, file=sys.stderr)
         except:
             import traceback;traceback.print_exc()
             failures.append((key, module))
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -31,6 +31,8 @@
 
 POSIX_EXE = 'pypy3'
 
+POSIX_EXE = 'pypy'
+
 from pypy.tool.build_cffi_imports import (create_cffi_import_libraries,
         MissingDependenciesError, cffi_build_scripts)
 
@@ -72,7 +74,7 @@
 
     basedir = py.path.local(basedir)
     if not override_pypy_c:
-        basename = 'pypy3-c'
+        basename = POSIX_EXE + '-c'
         if sys.platform == 'win32':
             basename += '.exe'
         pypy_c = basedir.join('pypy', 'goal', basename)
@@ -107,10 +109,13 @@
 
     if (sys.platform != 'win32' and    # handled below
         not _fake and os.path.getsize(str(pypy_c)) < 500000):
-        # This pypy3-c is very small, so it means it relies on libpypy3_c.so.
+        # This 'pypy_c' is very small, so it means it relies on a so/dll
         # If it would be bigger, it wouldn't.  That's a hack.
-        libpypy_name = ('libpypy3-c.so' if not sys.platform.startswith('darwin')
-                                        else 'libpypy3-c.dylib')
+        if sys.platform.startswith('darwin'):
+            ext = 'dylib'
+        else:
+            ext = 'so'
+        libpypy_name = 'lib' + POSIX_EXE + '-c.' + ext
         libpypy_c = pypy_c.new(basename=libpypy_name)
         if not libpypy_c.check():
             raise PyPyCNotFound('Expected pypy to be mostly in %r, but did '
@@ -135,8 +140,9 @@
             tgt = py.path.local(tgt)
             binaries.append((pypyw, tgt.new(purebasename=tgt.purebasename + 'w').basename, None))
             print("Picking %s" % str(pypyw))
-        # Can't rename a DLL: it is always called 'libpypy3-c.dll'
-        win_extras = [('libpypy3-c.dll', None), ('sqlite3.dll', lib_pypy)]
+        # Can't rename a DLL
+        win_extras = [('lib' + POSIX_EXE + '-c.dll', None),
+                      ('sqlite3.dll', lib_pypy)]
         if not options.no_tk:
             tkinter_dir = lib_pypy.join('_tkinter')
             win_extras += [('tcl85.dll', tkinter_dir), ('tk85.dll', tkinter_dir)]
@@ -156,9 +162,7 @@
             print('Picking %s (and contents)' % libsdir)
             shutil.copytree(str(libsdir), str(pypydir.join('libs')))
         else:
-            print('"libs" dir with import library not found.')
-            print('You have to create %r' % (str(libsdir),))
-            print('and copy libpypy3-c.lib in there, renamed to python32.lib')
+            raise RuntimError('"libs" dir with import library not found.')
             # XXX users will complain that they cannot compile capi (cpyext)
             # modules for windows, also embedding pypy (i.e. in cffi)
             # will fail.
@@ -186,14 +190,14 @@
                           for source, target, target_dir in binaries])
 
     # Careful: to copy lib_pypy, copying just the hg-tracked files
-    # would not be enough: there are also ctypes_config_cache/_*_cache.py.
-    # XXX ^^^ this is no longer true!
+    # would not be enough: there are also build artifacts like cffi-generated
+    # dynamic libs
     shutil.copytree(str(basedir.join('lib-python').join(STDLIB_VER)),
                     str(pypydir.join('lib-python').join(STDLIB_VER)),
                     ignore=ignore_patterns('.svn', 'py', '*.pyc', '*~'))
     shutil.copytree(str(basedir.join('lib_pypy')), str(lib_pypy),
                     ignore=ignore_patterns('.svn', 'py', '*.pyc', '*~',
-                                           '*_cffi.c', '*.o'))
+                                           '*_cffi.c', '*.o', '*.pyd-*'))
     for file in ['README.rst',]:
         shutil.copy(str(basedir.join(file)), str(pypydir))
     for file in ['_testcapimodule.c', '_ctypes_test.c']:
@@ -301,7 +305,7 @@
             setattr(ns, self.dest, option[2:4] != 'no')
 
     if sys.platform == 'win32':
-        pypy_exe = 'pypy3.exe'
+        pypy_exe = POSIX_EXE + '.exe'
     else:
         pypy_exe = POSIX_EXE
     parser = argparse.ArgumentParser()

From pypy.commits at gmail.com  Wed Nov 13 10:42:54 2019
From: pypy.commits at gmail.com (mattip)
Date: Wed, 13 Nov 2019 07:42:54 -0800 (PST)
Subject: [pypy-commit] buildbot default: add a step to purge hg files via
 command line trickery on windows
Message-ID: <5dcc247e.1c69fb81.eded8.e4ff@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r1102:4a51f6392247
Date: 2019-11-13 10:42 -0500
http://bitbucket.org/pypy/buildbot/changeset/4a51f6392247/

Log:	add a step to purge hg files via command line trickery on windows

diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py
--- a/bot2/pypybuildbot/builds.py
+++ b/bot2/pypybuildbot/builds.py
@@ -319,6 +319,14 @@
                              workdir=workdir,
                              timeout=3600,
                              haltOnFailure=True))
+    if platform == "win32":
+        # Clean out files via DOS to avoid long filename limitations in hg
+        command = 'for /F "usebackq tokens=1,2" %I IN (`hg stat`) DO @IF "?" == "%I" DEL %J'
+        factory.addStep(
+            ShellCmd(description="clean up files",
+                     command=command,
+                     workdir=workdir,
+                     haltOnFailure=True))
     #
     factory.addStep(
         ShellCmd(description="hg purge",

From pypy.commits at gmail.com  Wed Nov 13 11:09:41 2019
From: pypy.commits at gmail.com (mattip)
Date: Wed, 13 Nov 2019 08:09:41 -0800 (PST)
Subject: [pypy-commit] buildbot default: move build step to outer function
Message-ID: <5dcc2ac5.1c69fb81.e28e1.d8ae@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r1103:16b15bf68743
Date: 2019-11-13 09:09 -0700
http://bitbucket.org/pypy/buildbot/changeset/16b15bf68743/

Log:	move build step to outer function

diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py
--- a/bot2/pypybuildbot/builds.py
+++ b/bot2/pypybuildbot/builds.py
@@ -319,14 +319,6 @@
                              workdir=workdir,
                              timeout=3600,
                              haltOnFailure=True))
-    if platform == "win32":
-        # Clean out files via DOS to avoid long filename limitations in hg
-        command = 'for /F "usebackq tokens=1,2" %I IN (`hg stat`) DO @IF "?" == "%I" DEL %J'
-        factory.addStep(
-            ShellCmd(description="clean up files",
-                     command=command,
-                     workdir=workdir,
-                     haltOnFailure=True))
     #
     factory.addStep(
         ShellCmd(description="hg purge",
@@ -350,6 +342,15 @@
         update_hg_old_method(platform, factory, repourl, workdir, revision)
         return
 
+    if platform == "win32":
+        # Clean out files via DOS to avoid long filename limitations in hg
+        command = 'for /F "usebackq tokens=1,2" %I IN (`hg stat`) DO @IF "?" == "%I" DEL %J'
+        factory.addStep(
+            ShellCmd(description="clean up files",
+                     command=command,
+                     workdir=workdir,
+                     haltOnFailure=True))
+
     if wipe_bookmarks:
         # We don't use bookmarks at all.  If a bookmark accidentally gets
         # created and pushed to the server and we pull it down, it gets stuck

From pypy.commits at gmail.com  Wed Nov 13 11:43:00 2019
From: pypy.commits at gmail.com (mattip)
Date: Wed, 13 Nov 2019 08:43:00 -0800 (PST)
Subject: [pypy-commit] buildbot default: try 4a51f6392247 again but
 differently (arigato)
Message-ID: <5dcc3294.1c69fb81.78281.f0dd@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r1104:4221429ae63d
Date: 2019-11-13 09:42 -0700
http://bitbucket.org/pypy/buildbot/changeset/4221429ae63d/

Log:	try 4a51f6392247 again but differently (arigato)

diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py
--- a/bot2/pypybuildbot/builds.py
+++ b/bot2/pypybuildbot/builds.py
@@ -343,8 +343,8 @@
         return
 
     if platform == "win32":
-        # Clean out files via DOS to avoid long filename limitations in hg
-        command = 'for /F "usebackq tokens=1,2" %I IN (`hg stat`) DO @IF "?" == "%I" DEL %J'
+        # Clean out files via hackery to avoid long filename limitations in hg
+        command = 'hg update -r null & rmdir /q /s lib_pypy extra_tests pypy'
         factory.addStep(
             ShellCmd(description="clean up files",
                      command=command,

From pypy.commits at gmail.com  Wed Nov 13 12:53:42 2019
From: pypy.commits at gmail.com (mattip)
Date: Wed, 13 Nov 2019 09:53:42 -0800 (PST)
Subject: [pypy-commit] pypy py3.6: fix bad merge
Message-ID: <5dcc4326.1c69fb81.49661.ed08@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r98046:e04884ede379
Date: 2019-11-13 10:52 -0700
http://bitbucket.org/pypy/pypy/changeset/e04884ede379/

Log:	fix bad merge

diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -26,13 +26,9 @@
 from pypy.tool.release.make_portable import make_portable
 
 USE_ZIPFILE_MODULE = sys.platform == 'win32'
-
 STDLIB_VER = "3"
-
 POSIX_EXE = 'pypy3'
 
-POSIX_EXE = 'pypy'
-
 from pypy.tool.build_cffi_imports import (create_cffi_import_libraries,
         MissingDependenciesError, cffi_build_scripts)
 

From pypy.commits at gmail.com  Wed Nov 13 12:58:46 2019
From: pypy.commits at gmail.com (wlav)
Date: Wed, 13 Nov 2019 09:58:46 -0800 (PST)
Subject: [pypy-commit] pypy cppyy-dev: pointers to unknown classes now work
 (but are otherwise not usable)
Message-ID: <5dcc4456.1c69fb81.463aa.45be@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-dev
Changeset: r98047:3d97f3c4624e
Date: 2019-11-07 19:45 -0800
http://bitbucket.org/pypy/pypy/changeset/3d97f3c4624e/

Log:	pointers to unknown classes now work (but are otherwise not usable)

diff --git a/pypy/module/_cppyy/test/test_fragile.py b/pypy/module/_cppyy/test/test_fragile.py
--- a/pypy/module/_cppyy/test/test_fragile.py
+++ b/pypy/module/_cppyy/test/test_fragile.py
@@ -35,7 +35,7 @@
 
         assert fragile.B == fragile.B
         assert fragile.B().check() == ord('B')
-        raises(AttributeError, getattr, fragile.B().gime_no_such(), "__cppdecl__")
+        assert not fragile.B().gime_no_such()
 
         assert fragile.C == fragile.C
         assert fragile.C().check() == ord('C')

From pypy.commits at gmail.com  Wed Nov 13 12:58:48 2019
From: pypy.commits at gmail.com (wlav)
Date: Wed, 13 Nov 2019 09:58:48 -0800 (PST)
Subject: [pypy-commit] pypy cppyy-dev: extend support of templated methods
Message-ID: <5dcc4458.1c69fb81.ecab8.e397@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-dev
Changeset: r98048:65d87222eef1
Date: 2019-11-13 09:49 -0800
http://bitbucket.org/pypy/pypy/changeset/65d87222eef1/

Log:	extend support of templated methods

diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -1355,6 +1355,8 @@
     def _build_overloads(self):
         assert len(self.overloads) == 0
         methods_tmp = {}; ftype_tmp = {}
+
+        # add all ordinary methods (incl. pre-instantiated templates)
         for idx in range(capi.c_num_methods(self.space, self)):
             cppmeth = capi.c_get_method(self.space, self, idx)
             if capi.c_is_constructor(self.space, cppmeth):
@@ -1372,6 +1374,7 @@
             ftype_tmp[pyname] |= self._make_cppfunction(pyname, cppmeth, methods)
             if capi.c_method_is_template(self.space, self, idx):
                 ftype_tmp[pyname] |= FUNCTION_IS_TEMPLATE
+
         # the following covers the case where the only kind of operator[](idx)
         # returns are the ones that produce non-const references; these can be
         # used for __getitem__ just as much as for __setitem__, though
@@ -1407,6 +1410,11 @@
                 overload = W_CPPOverload(self.space, self, methods[:])
             self.overloads[pyname] = overload
 
+        # add placeholders for all non-instantiated templated methods
+        for idx in range(capi.c_get_num_templated_methods(self.space, self)):
+            cppname = capi.c_get_templated_method_name(self.space, self, idx)
+            self.overloads[cppname] = W_CPPTemplateOverload(self.space, cppname, None, self, [])
+
     def _make_cppfunction(self, pyname, cppmeth, funcs):
         num_args = capi.c_method_num_args(self.space, cppmeth)
         args_required = capi.c_method_req_args(self.space, cppmeth)

From pypy.commits at gmail.com  Wed Nov 13 17:00:22 2019
From: pypy.commits at gmail.com (wlav)
Date: Wed, 13 Nov 2019 14:00:22 -0800 (PST)
Subject: [pypy-commit] pypy cppyy-dev: prepare for more template tests
Message-ID: <5dcc7cf6.1c69fb81.40427.4cad@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-dev
Changeset: r98049:47555bb4c1cc
Date: 2019-11-13 11:50 -0800
http://bitbucket.org/pypy/pypy/changeset/47555bb4c1cc/

Log:	prepare for more template tests

diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -981,8 +981,8 @@
 )
 
 class W_CPPTemplateStaticOverload(W_CPPStaticOverload, TemplateOverloadMixin):
-    """App-level dispatcher to allow both lookup/instantiation of templated methods and
-    dispatch among overloads between templated and non-templated method."""
+    """Dispatcher to allow both lookup/instantiation of templated methods and
+    select among templated and non-templated method overloads."""
 
     _attrs_ = ['name', 'tmpl_args', 'overloads', 'master', 'w_this']
     _immutable_fields_ = ['name', 'tmpl_args']
@@ -996,7 +996,8 @@
          self.w_this = space.w_None
 
     def clone(self, tmpl_args):
-        other = W_CPPTemplateStaticOverload(self.space, self.name, tmpl_args, self.scope, self.functions, self.flags)
+        other = W_CPPTemplateStaticOverload(self.space, self.name,
+            tmpl_args, self.scope, self.functions, self.flags)
         other.overloads = self.overloads
         other.master = self.master
         other.w_this = self.w_this
@@ -1007,7 +1008,8 @@
         if isinstance(w_cppinstance, W_CPPInstance):
             cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance)
             if cppinstance.clsdecl.handle != self.scope.handle:
-                cppol = W_CPPTemplateStaticOverload(self.space, self.name, self.tmpl_args, self.scope, self.functions, self.flags)
+                cppol = W_CPPTemplateStaticOverload(self.space, self.name,
+                    self.tmpl_args, self.scope, self.functions, self.flags)
                 cppol.w_this = w_cppinstance
                 cppol.master = self.master
                 return cppol       # bound
@@ -1256,10 +1258,10 @@
         return self.handle != other.handle
 
 
-# For now, keep namespaces and classes separate as namespaces are extensible
+# Namespaces and classes are separate as namespaces are (more) extensible
 # with info from multiple dictionaries and do not need to bother with meta
-# classes for inheritance. Both are python classes, though, and refactoring
-# may be in order at some point.
+# classes for inheritance. Both are python classes, though, and further
+# refactoring may be in order at some point.
 class W_CPPNamespaceDecl(W_CPPScopeDecl):
     _attrs_ = ['space', 'handle', 'name', 'overloads', 'datamembers']
     _immutable_fields_ = ['handle', 'name']
diff --git a/pypy/module/_cppyy/test/templates.cxx b/pypy/module/_cppyy/test/templates.cxx
--- a/pypy/module/_cppyy/test/templates.cxx
+++ b/pypy/module/_cppyy/test/templates.cxx
@@ -10,3 +10,28 @@
 long MyTemplatedMethodClass::get_float_size()  { return (long)sizeof(float); }
 long MyTemplatedMethodClass::get_double_size() { return (long)sizeof(double); }
 long MyTemplatedMethodClass::get_self_size()   { return (long)sizeof(MyTemplatedMethodClass); }
+
+
+// variadic templates
+#ifdef WIN32
+__declspec(dllexport)
+#endif
+std::string some_variadic::gTypeName = "";
+
+
+// template with empty body
+namespace T_WithEmptyBody {
+
+#ifdef WIN32
+__declspec(dllexport)
+#endif
+std::string side_effect = "not set";
+
+template<typename T>
+void some_empty() {
+    side_effect = "side effect";
+}
+
+template void some_empty<int>();
+
+} // namespace T_WithRValue
diff --git a/pypy/module/_cppyy/test/templates.h b/pypy/module/_cppyy/test/templates.h
--- a/pypy/module/_cppyy/test/templates.h
+++ b/pypy/module/_cppyy/test/templates.h
@@ -1,12 +1,33 @@
+#ifndef CPPYY_TEST_TEMPLATES_H
+#define CPPYY_TEST_TEMPLATES_H
+
+#include <stdexcept>
 #include <string>
 #include <sstream>
 #include <vector>
 
+#ifndef WIN32
+#include <cxxabi.h>
+inline std::string demangle_it(const char* name, const char* errmsg) {
+    int status;
+    std::string res = abi::__cxa_demangle(name, 0, 0, &status);
+    if (status != 0) throw std::runtime_error(errmsg);
+    return res;
+}
+#else
+inline std::string demangle_it(const char* name, const char*) {
+    return name;        // typeinfo's name() is already demangled
+}
+#endif
+
 
 //===========================================================================
 class MyTemplatedMethodClass {         // template methods
 public:
-    long get_size();      // to get around bug in genreflex
+    template<class A> long get_size(A&);
+    template<class A> long get_size(const A&);
+
+    long get_size();
     template<class B> long get_size();
 
     long get_char_size();
@@ -21,6 +42,16 @@
     double m_data[3];
 };
 
+template<class A>
+long MyTemplatedMethodClass::get_size(A&) {
+    return sizeof(A);
+}
+
+template<class A>
+long MyTemplatedMethodClass::get_size(const A&) {
+    return sizeof(A)+1;
+}
+
 template<class B>
 inline long MyTemplatedMethodClass::get_size() {
     return sizeof(B);
@@ -63,7 +94,7 @@
 };
 
 template <class I, typename O = float>
-SomeResult<O> global_get_some_result(const std::vector<I>& carrier) {
+SomeResult<O> global_get_some_result(const I& carrier) {
     SomeResult<O> r{};
     r.m_retval = O(carrier[0]);
     return r;
@@ -156,3 +187,295 @@
     T m_value;
 };
 */
+
+
+//===========================================================================
+// templated callable
+class TemplatedCallable {
+public:
+    template <class I , class O = double>
+    O operator() (const I& in) const { return O(in); }
+};
+
+
+//===========================================================================
+// templated typedefs
+namespace TemplatedTypedefs {
+
+template<typename TYPE_IN, typename TYPE_OUT, size_t _vsize = 4>
+struct BaseWithEnumAndTypedefs {
+    enum { vsize = _vsize };
+    typedef TYPE_IN in_type;
+    typedef TYPE_OUT out_type;
+};
+
+template <typename TYPE_IN, typename TYPE_OUT, size_t _vsize = 4>
+struct DerivedWithUsing : public BaseWithEnumAndTypedefs<TYPE_IN, TYPE_OUT, _vsize>
+{
+    typedef BaseWithEnumAndTypedefs<TYPE_IN, TYPE_OUT, _vsize> base_type;
+    using base_type::vsize;
+    using typename base_type::in_type;
+    typedef typename base_type::in_type in_type_tt;
+    using typename base_type::out_type;
+};
+
+struct SomeDummy {};
+
+} // namespace TemplatedTypedefs
+
+
+//===========================================================================
+// hiding templated methods
+namespace TemplateHiding {
+
+struct Base {
+    template<class T>
+    int callme(T t = T(1)) { return 2*t; }
+};
+
+struct Derived : public Base {
+    int callme(int t = 2) { return t; }
+};
+
+} // namespace TemplateHiding
+
+
+//===========================================================================
+// 'using' of templates
+template<typename T> using DA_vector = std::vector<T>;
+
+#if __cplusplus > 201402L
+namespace using_problem {
+
+template <typename T, size_t SZ>
+struct vector {
+    vector() : m_val(SZ) {}
+    T m_val;
+};
+
+template <typename T, size_t ... sizes>
+struct matryoshka {
+    typedef T type;
+};
+
+template <typename T, size_t SZ, size_t ... sizes>
+struct matryoshka<T, SZ, sizes ... > {
+    typedef vector<typename matryoshka<T, sizes ...>::type, SZ> type;
+};
+
+template <typename T, size_t ... sizes>
+using make_vector = typename matryoshka<T, sizes ...>::type;
+    typedef make_vector<int, 2, 3> iiv_t;
+};
+#endif
+
+namespace using_problem {
+
+template<typename T>
+class Base {
+public:
+    template<typename R>
+    R get1(T t) { return t + R{5}; }
+    T get2() { return T{5}; }
+    template<typename R>
+    R get3(T t) { return t + R{5}; }
+    T get3() { return T{5}; }
+};
+
+template<typename T>
+class Derived : public Base<T> {
+public:
+    typedef Base<T> _Mybase;
+    using _Mybase::get1;
+    using _Mybase::get2;
+    using _Mybase::get3;
+};
+
+} // namespace using_problem
+
+
+//===========================================================================
+// template with r-value
+namespace T_WithRValue {
+
+template<typename T>
+bool is_valid(T&& new_value) {
+    return new_value != T{};
+}
+
+} // namespace T_WithRValue
+
+
+//===========================================================================
+// variadic templates
+namespace some_variadic {
+
+#ifdef WIN32
+extern __declspec(dllimport) std::string gTypeName;
+#else
+extern std::string gTypeName;
+#endif
+
+template <typename ... Args>
+class A {
+public:
+    A() {
+        gTypeName = demangle_it(typeid(A<Args...>).name(), "A::A");
+    }
+    A(const A&) = default;
+    A(A&&) = default;
+    A& operator=(const A&) = default;
+    A& operator=(A&&) = default;
+
+    template <typename ... FArgs>
+    void a(FArgs&&... args) {
+        gTypeName = demangle_it(typeid(&A<Args...>::a<FArgs...>).name(), "A::a-2");
+    }
+
+    template <typename T, typename ... FArgs>
+    T a_T(FArgs&&... args) {
+        gTypeName = demangle_it(typeid(&A<Args...>::a_T<T, FArgs...>).name(), "A::a_T-2");
+        return T{};
+    }
+
+    template <typename ... FArgs>
+    static void sa(FArgs&&... args) {
+        gTypeName = demangle_it(typeid(A<Args...>).name(), "A::sa-1");
+        gTypeName += "::";
+        gTypeName += demangle_it(typeid(A<Args...>::sa<FArgs...>).name(), "A::sa-2");
+    }
+
+    template <typename T, typename ... FArgs>
+    static T sa_T(FArgs&&... args) {
+        gTypeName = demangle_it(typeid(A<Args...>).name(), "A::sa_T-1");
+        gTypeName +=  "::";
+        gTypeName += demangle_it(typeid(A<Args...>::sa_T<T, FArgs...>).name(), "A::sa_T-2");
+        return T{};
+    }
+};
+
+class B {
+public:
+    B() {
+        gTypeName = demangle_it(typeid(B).name(), "B::B");
+    }
+    B(const B&) = default;
+    B(B&&) = default;
+    B& operator=(const B&) = default;
+    B& operator=(B&&) = default;
+
+    template <typename ... FArgs>
+    void b(FArgs&&... args) {
+        gTypeName = demangle_it(typeid(&B::b<FArgs...>).name(), "B::b-2");
+    }
+
+    template <typename T, typename ... FArgs>
+    T b_T(FArgs&&... args) {
+        gTypeName = demangle_it(typeid(&B::b_T<T, FArgs...>).name(), "B::b_T-2");
+        return T{};
+    }
+
+    template <typename ... FArgs>
+    static void sb(FArgs&&... args) {
+        gTypeName = demangle_it(typeid(B).name(), "B::sb-1");
+        gTypeName += "::";
+        gTypeName +=  demangle_it(typeid(B::sb<FArgs...>).name(), "B::sb-2");
+    }
+
+    template <typename T, typename ... FArgs>
+    static T sb_T(FArgs&&... args) {
+        gTypeName = demangle_it(typeid(B).name(), "B::sb_T-1");
+        gTypeName += "::";
+        gTypeName += demangle_it(typeid(B::sb_T<T, FArgs...>).name(), "B::sb_T-2");
+        return T{};
+    }
+};
+
+template <typename ... Args>
+void fn(Args&&... args) {
+    gTypeName = demangle_it(typeid(fn<Args...>).name(), "fn");
+}
+
+template <typename T, typename ... Args>
+T fn_T(Args&&... args) {
+    gTypeName = demangle_it(typeid(fn<Args...>).name(), "fn_T");
+    return T{};
+}
+
+} // namespace some_variadic
+
+
+//===========================================================================
+// template with empty body
+namespace T_WithEmptyBody {
+
+#ifdef WIN32
+extern __declspec(dllimport) std::string side_effect;
+#else
+extern std::string side_effect;
+#endif
+
+template<typename T>
+void some_empty();
+
+} // namespace T_WithEmptyBody
+
+
+//===========================================================================
+// template with catch-all (void*, void**)overloads
+namespace T_WithGreedyOverloads {
+
+class SomeClass {
+    double fD;
+};
+
+class WithGreedy1 {
+public:
+    template<class T>
+    int get_size(T*) { return (int)sizeof(T); }
+    int get_size(void*, bool force=false) { return -1; }
+};
+
+class WithGreedy2 {
+public:
+    template<class T>
+    int get_size(T*) { return (int)sizeof(T); }
+    int get_size(void**, bool force=false) { return -1; }
+};
+
+class DoesNotExist;
+
+class WithGreedy3 {
+public:
+    template<class T>
+    int get_size(T*) { return (int)sizeof(T); }
+    int get_size(DoesNotExist*, bool force=false) { return -1; }
+};
+
+} // namespace T_WithGreedyOverloads
+
+
+//===========================================================================
+// template with overloaded non-templated and templated setitem
+namespace TemplateWithSetItem {
+
+template <typename T>
+class MyVec {
+private:
+   std::vector<T> fData;
+
+public:
+   using size_type = typename std::vector<T>::size_type;
+
+   MyVec(size_type count) : fData(count) {}
+
+   T & operator[](size_type index) { return fData[index]; }
+
+   // The definition of this templated operator causes the issue
+   template <typename V>
+   MyVec operator[](const MyVec<V> &conds) const { return MyVec(2); }
+};
+
+} // namespace TemplateWithSetItem
+
+#endif // !CPPYY_TEST_TEMPLATES_H
diff --git a/pypy/module/_cppyy/test/templates.xml b/pypy/module/_cppyy/test/templates.xml
--- a/pypy/module/_cppyy/test/templates.xml
+++ b/pypy/module/_cppyy/test/templates.xml
@@ -16,4 +16,21 @@
   <function name="SomeNS::some_bar" />
   <function name="SomeNS::tuplify" />
 
+  <class name="TemplatedCallable" />
+
+  <namespace name="TemplateHiding" />
+  <class pattern="TemplateHiding::*" />
+
+  <namespace name="using_problem" />
+
+  <namespace name="some_variadic" />
+  <class pattern="some_variadic::*" />
+  <function pattern="some_variadic::*" />
+  <variable pattern="some_variadic::*" />
+
+  <namespace name="T_WithEmptyBody" />
+
+  <namespace name="T_WithGreedyOverloads" />
+  <class pattern="T_WithGreedyOverloads::*" />
+
 </lcgdict>

From pypy.commits at gmail.com  Wed Nov 13 17:00:24 2019
From: pypy.commits at gmail.com (wlav)
Date: Wed, 13 Nov 2019 14:00:24 -0800 (PST)
Subject: [pypy-commit] pypy cppyy-dev: more template tests from cppyy/test
 (CPython)
Message-ID: <5dcc7cf8.1c69fb81.f45e4.3a5f@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-dev
Changeset: r98050:5e3e6dd38086
Date: 2019-11-13 13:59 -0800
http://bitbucket.org/pypy/pypy/changeset/5e3e6dd38086/

Log:	more template tests from cppyy/test (CPython)

diff --git a/pypy/module/_cppyy/test/test_templates.py b/pypy/module/_cppyy/test/test_templates.py
--- a/pypy/module/_cppyy/test/test_templates.py
+++ b/pypy/module/_cppyy/test/test_templates.py
@@ -25,20 +25,35 @@
 
         m = _cppyy.gbl.MyTemplatedMethodClass()
 
+      # implicit (called before other tests to check caching)
+        assert m.get_size(1)          == m.get_int_size()+1
+        assert 'get_size<int>' in dir(_cppyy.gbl.MyTemplatedMethodClass)
+
       # pre-instantiated
         assert m.get_size['char']()   == m.get_char_size()
         assert m.get_size[int]()      == m.get_int_size()
 
       # specialized
-        assert m.get_size[long]()     == m.get_long_size()
+        if sys.hexversion >= 0x3000000:
+            targ = 'long'
+        else:
+            targ = long
+        assert m.get_size[targ]()     == m.get_long_size()
+
+        import ctypes
+        assert m.get_size(ctypes.c_double(3.14)) == m.get_size['double']()
+        assert m.get_size(ctypes.c_double(3.14).value) == m.get_size['double']()+1
 
       # auto-instantiation
         assert m.get_size[float]()    == m.get_float_size()
         assert m.get_size['double']() == m.get_double_size()
         assert m.get_size['MyTemplatedMethodClass']() == m.get_self_size()
+        assert 'get_size<MyTemplatedMethodClass>' in dir(_cppyy.gbl.MyTemplatedMethodClass)
 
       # auto through typedef
         assert m.get_size['MyTMCTypedef_t']() == m.get_self_size()
+        assert 'get_size<MyTMCTypedef_t>' in dir(_cppyy.gbl.MyTemplatedMethodClass)
+        assert m.get_size['MyTemplatedMethodClass']() == m.get_self_size()
 
     def test02_non_type_template_args(self):
         """Use of non-types as template arguments"""
@@ -90,15 +105,13 @@
         assert type(ggsr(vector['int']([5])).m_retval) == float
         assert ggsr(vector['int']([5])).m_retval == 5.
         # float in, int out
-        # TODO: this now matches the earlier overload
-        #ggsr = cppyy.gbl.global_get_some_result['std::vector<float>, int']
-        #assert type(ggsr(vector['float']([0.3])).m_retval) == int
-        #assert ggsr(vector['float']([0.3])).m_retval == 0
+        ggsr = cppyy.gbl.global_get_some_result['std::vector<float>, int']
+        assert type(ggsr(vector['float']([0.3])).m_retval) == int
+        assert ggsr(vector['float']([0.3])).m_retval == 0
         # int in, int out
-        # TODO: same as above, matches earlier overload
-        #ggsr = cppyy.gbl.global_get_some_result['std::vector<int>, int']
-        #assert type(ggsr(vector['int']([5])).m_retval) == int
-        #assert ggsr(vector['int']([5])).m_retval == 5
+        ggsr = cppyy.gbl.global_get_some_result['std::vector<int>, int']
+        assert type(ggsr(vector['int']([5])).m_retval) == int
+        assert ggsr(vector['int']([5])).m_retval == 5
 
     def test04_variadic_function(self):
         """Call a variadic function"""

From pypy.commits at gmail.com  Wed Nov 13 22:08:25 2019
From: pypy.commits at gmail.com (mattip)
Date: Wed, 13 Nov 2019 19:08:25 -0800 (PST)
Subject: [pypy-commit] buildbot default: don't fail if there are locked files
Message-ID: <5dccc529.1c69fb81.67b57.852d@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r1105:7d54cdb57949
Date: 2019-11-13 20:08 -0700
http://bitbucket.org/pypy/buildbot/changeset/7d54cdb57949/

Log:	don't fail if there are locked files

diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py
--- a/bot2/pypybuildbot/builds.py
+++ b/bot2/pypybuildbot/builds.py
@@ -349,7 +349,7 @@
             ShellCmd(description="clean up files",
                      command=command,
                      workdir=workdir,
-                     haltOnFailure=True))
+                     haltOnFailure=False))
 
     if wipe_bookmarks:
         # We don't use bookmarks at all.  If a bookmark accidentally gets

From pypy.commits at gmail.com  Thu Nov 14 09:28:53 2019
From: pypy.commits at gmail.com (mattip)
Date: Thu, 14 Nov 2019 06:28:53 -0800 (PST)
Subject: [pypy-commit] pypy py3.6: pass 'final' down to where it is handled,
 add error check
Message-ID: <5dcd64a5.1c69fb81.409da.18e7@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r98051:3eabce9188ed
Date: 2019-11-14 09:27 -0500
http://bitbucket.org/pypy/pypy/changeset/3eabce9188ed/

Log:	pass 'final' down to where it is handled, add error check

diff --git a/pypy/interpreter/unicodehelper_win32.py b/pypy/interpreter/unicodehelper_win32.py
--- a/pypy/interpreter/unicodehelper_win32.py
+++ b/pypy/interpreter/unicodehelper_win32.py
@@ -41,13 +41,13 @@
         return 0
     return rwin32.WC_NO_BEST_FIT_CHARS
 
-def _decode_cp_error(s, errorhandler, encoding, errors, start, end):
+def _decode_cp_error(s, errorhandler, encoding, errors, final, start, end):
     # late import to avoid circular import
     from pypy.interpreter.unicodehelper import _str_decode_utf8_slowpath
     if rwin32.GetLastError_saved() == rwin32.ERROR_NO_UNICODE_TRANSLATION:
         msg = ("No mapping for the Unicode character exists in the target "
                "multi-byte code page.")
-        r, ignore1, ignore2 = _str_decode_utf8_slowpath(s[start:end], errors, False, errorhandler, False)
+        r, ignore1, ignore2 = _str_decode_utf8_slowpath(s[start:end], errors, final, errorhandler, False)
         return r, end
     else:
         raise rwin32.lastSavedWindowsError()
@@ -77,7 +77,7 @@
             return result
 
 def _decode_helper(cp, s, flags, encoding, errors, errorhandler, 
-                   start, end, res):
+                   final, start, end, res):
     if end > len(s):
         end = len(s)
     piece = s[start:end]
@@ -87,7 +87,7 @@
                                     lltype.nullptr(rffi.CWCHARP.TO), 0)
         if outsize == 0:
             r, pos = _decode_cp_error(s, errorhandler,
-                                               encoding, errors, start, end)
+                                           encoding, errors, final, start, end)
             res.append(r)
             return pos
 
@@ -96,7 +96,7 @@
             if MultiByteToWideChar(cp, flags, dataptr, len(piece),
                                    buf.raw, outsize) == 0:
                 r, pos = _decode_cp_error(s, errorhandler,
-                                             encoding, errors, start, end)
+                                           encoding, errors, final, start, end)
                 res.append(r)
                 return pos
             else:
@@ -116,14 +116,14 @@
     res = StringBuilder(insize)
     if errors == 'strict':
         _decode_helper(cp, s, flags, encoding, errors, errorhandler,
-                       0, len(s), res)
+                       final, 0, len(s), res)
     else:
         prev_pos = 0
         pos = 0
         while pos < len(s):
             pos = next_codepoint_pos(s, prev_pos)
-            pos = _decode_helper(cp, s, flags, encoding,
-                            errors, errorhandler, prev_pos, pos, res)
+            pos = _decode_helper(cp, s, flags, encoding, errors,
+                                 errorhandler, final, prev_pos, pos, res)
             prev_pos = pos
     return res.build(), insize
 
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -716,6 +716,9 @@
     @unwrap_spec(code_page=int, errors='text_or_none')
     def code_page_encode(space, code_page, w_arg, errors="strict"):
         # w_arg is a W_Unicode or W_Bytes?
+        if code_page < 0:
+            raise oefmt(space.w_ValueError, "invalid code page number %d",
+                        code_page)
         w_arg = space.convert_arg_to_w_unicode(w_arg, errors)
         if errors is None:
             errors = 'strict'
@@ -732,6 +735,9 @@
     @unwrap_spec(code_page=int, string='bufferstr', errors='text_or_none',
                  w_final=WrappedDefault(False))
     def code_page_decode(space, code_page, string, errors="strict", w_final=None):
+        if code_page < 0:
+            raise oefmt(space.w_ValueError, "invalid code page number %d",
+                        code_page)
         if errors is None:
             errors = 'strict'
         final = space.is_true(w_final)

From pypy.commits at gmail.com  Thu Nov 14 16:59:58 2019
From: pypy.commits at gmail.com (mattip)
Date: Thu, 14 Nov 2019 13:59:58 -0800 (PST)
Subject: [pypy-commit] pypy py3.6: catch an OSError raised when {en,
 de}coding with a codepage
Message-ID: <5dcdce5e.1c69fb81.b0e75.7039@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r98053:2769fb7dff63
Date: 2019-11-14 14:59 -0700
http://bitbucket.org/pypy/pypy/changeset/2769fb7dff63/

Log:	catch an OSError raised when {en,de}coding with a codepage

diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -5,7 +5,7 @@
 from rpython.rlib.rutf8 import MAXUNICODE
 from rpython.rlib import runicode
 
-from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.error import OperationError, oefmt, wrap_oserror
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
 from pypy.interpreter import unicodehelper
 from pypy.module.unicodedata.interp_ucd import unicodedb
@@ -727,9 +727,12 @@
             allow_surrogates = True
         state = space.fromcache(CodecState)
         ulen = w_arg._length
-        result = unicodehelper.utf8_encode_code_page(code_page, w_arg._utf8,
+        try:
+            result = unicodehelper.utf8_encode_code_page(code_page, w_arg._utf8,
                       errors, state.encode_error_handler,
                       allow_surrogates=allow_surrogates)
+        except OSError as e:
+            raise wrap_oserror(space, e)
         return space.newtuple([space.newbytes(result), space.newint(ulen)])
 
     @unwrap_spec(code_page=int, string='bufferstr', errors='text_or_none',
@@ -742,9 +745,12 @@
             errors = 'strict'
         final = space.is_true(w_final)
         state = space.fromcache(CodecState)
-        result, length, pos = unicodehelper.str_decode_code_page(code_page,
+        try:
+            result, length, pos = unicodehelper.str_decode_code_page(code_page,
                                    string, errors, final,
                                    state.decode_error_handler)
+        except OSError as e:
+            raise wrap_oserror(space, e)
         # must return bytes, pos
         return space.newtuple([space.newutf8(result, length), space.newint(pos)])
 

From pypy.commits at gmail.com  Thu Nov 14 22:47:19 2019
From: pypy.commits at gmail.com (mattip)
Date: Thu, 14 Nov 2019 19:47:19 -0800 (PST)
Subject: [pypy-commit] pypy default: package.py: fix typo,
 condition exception on _fake=False
Message-ID: <5dce1fc7.1c69fb81.6633c.f451@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r98054:3ad0fbece144
Date: 2019-11-14 20:46 -0700
http://bitbucket.org/pypy/pypy/changeset/3ad0fbece144/

Log:	package.py: fix typo, condition exception on _fake=False

diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -161,7 +161,8 @@
             print('Picking %s (and contents)' % libsdir)
             shutil.copytree(str(libsdir), str(pypydir.join('libs')))
         else:
-            raise RuntimError('"libs" dir with import library not found.')
+            if not _fake:
+                raise RuntimeError('"libs" dir with import library not found.')
             # XXX users will complain that they cannot compile capi (cpyext)
             # modules for windows, also embedding pypy (i.e. in cffi)
             # will fail.

From pypy.commits at gmail.com  Fri Nov 15 07:17:08 2019
From: pypy.commits at gmail.com (mattip)
Date: Fri, 15 Nov 2019 04:17:08 -0800 (PST)
Subject: [pypy-commit] pypy default: wint_t is not defined in all versions
 of curses
Message-ID: <5dce9744.1c69fb81.58b60.9ca1@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r98055:e36c9b5c55a7
Date: 2019-11-15 07:15 -0500
http://bitbucket.org/pypy/pypy/changeset/e36c9b5c55a7/

Log:	wint_t is not defined in all versions of curses

diff --git a/lib_pypy/_curses_build.py b/lib_pypy/_curses_build.py
--- a/lib_pypy/_curses_build.py
+++ b/lib_pypy/_curses_build.py
@@ -96,8 +96,6 @@
 typedef unsigned long... chtype;
 typedef chtype attr_t;
 
-typedef int... wint_t;
-
 typedef struct
 {
     short id;           /* ID to distinguish multiple devices */
@@ -376,6 +374,7 @@
 
 if version > (5, 7):
     ffi.cdef("""
+typedef int... wint_t;
 int wget_wch(WINDOW *, wint_t *);
 int mvwget_wch(WINDOW *, int, int, wint_t *);
 int unget_wch(const wchar_t);

From pypy.commits at gmail.com  Fri Nov 15 07:52:13 2019
From: pypy.commits at gmail.com (mattip)
Date: Fri, 15 Nov 2019 04:52:13 -0800 (PST)
Subject: [pypy-commit] pypy default: fix 2c98ee4a95b3
Message-ID: <5dce9f7d.1c69fb81.c46cb.8891@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r98056:1ad4f44a62a4
Date: 2019-11-15 07:51 -0500
http://bitbucket.org/pypy/pypy/changeset/1ad4f44a62a4/

Log:	fix 2c98ee4a95b3

diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py
--- a/lib-python/2.7/sysconfig.py
+++ b/lib-python/2.7/sysconfig.py
@@ -383,9 +383,10 @@
     vars['VERSION'] = _PY_VERSION_SHORT_NO_DOT
     vars['BINDIR'] = os.path.dirname(_safe_realpath(sys.executable))
     # pypy only: give us control over the ABI tag in a wheel name
-    import _imp
-    so_ext = _imp.get_suffixes()[0][0]
-    vars['SOABI']= '-'.join(so_ext.split('.')[1].split('-')[:2])
+    if '__pypy__' in sys.builtin_module_names:
+        import imp
+        so_ext = imp.get_suffixes()[0][0]
+        vars['SOABI']= '-'.join(so_ext.split('.')[1].split('-')[:2])
 
 #
 # public APIs

From pypy.commits at gmail.com  Fri Nov 15 13:46:05 2019
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 15 Nov 2019 10:46:05 -0800 (PST)
Subject: [pypy-commit] pypy hpy: A branch to create an HPy implementation in
 PyPy.
Message-ID: <5dcef26d.1c69fb81.90305.cbd0@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: hpy
Changeset: r98057:a4ddff5d8e92
Date: 2019-11-15 19:37 +0100
http://bitbucket.org/pypy/pypy/changeset/a4ddff5d8e92/

Log:	A branch to create an HPy implementation in PyPy.


From pypy.commits at gmail.com  Fri Nov 15 13:50:06 2019
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 15 Nov 2019 10:50:06 -0800 (PST)
Subject: [pypy-commit] pypy hpy: Create empty hpy_universal module
Message-ID: <5dcef35e.1c69fb81.fe85c.af10@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: hpy
Changeset: r98058:6d77e45b6f40
Date: 2019-11-15 19:49 +0100
http://bitbucket.org/pypy/pypy/changeset/6d77e45b6f40/

Log:	Create empty hpy_universal module

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -40,7 +40,7 @@
     "binascii", "_multiprocessing", '_warnings', "_collections",
     "_multibytecodec", "_continuation", "_cffi_backend",
     "_csv", "_pypyjson", "_posixsubprocess", "_cppyy", # "micronumpy",
-    "_jitlog",
+    "_jitlog", "hpy_universal"
     # "_hashlib", "crypt"
 ])
 
diff --git a/pypy/module/hpy_universal/__init__.py b/pypy/module/hpy_universal/__init__.py
new file mode 100644
diff --git a/pypy/module/hpy_universal/moduledef.py b/pypy/module/hpy_universal/moduledef.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/moduledef.py
@@ -0,0 +1,5 @@
+from pypy.interpreter.mixedmodule import MixedModule
+
+class Module(MixedModule):
+    interpleveldefs ={}
+    appleveldefs = {}
diff --git a/pypy/module/hpy_universal/test/test_basic.py b/pypy/module/hpy_universal/test/test_basic.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/test/test_basic.py
@@ -0,0 +1,4 @@
+class AppTestBasic(object):
+    spaceconfig = {'usemodules': ['hpy_universal']}
+    def test_import(self):
+        import hpy_universal

From pypy.commits at gmail.com  Fri Nov 15 14:58:40 2019
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 15 Nov 2019 11:58:40 -0800 (PST)
Subject: [pypy-commit] pypy hpy: (antocuni,
 ronan) Setup the infrastructure to compile and test HPy modules
Message-ID: <5dcf0370.1c69fb81.9f190.bc70@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: hpy
Changeset: r98059:19d6e8fbace2
Date: 2019-11-15 20:57 +0100
http://bitbucket.org/pypy/pypy/changeset/19d6e8fbace2/

Log:	(antocuni, ronan) Setup the infrastructure to compile and test HPy
	modules

	Vendor files from pyhandle/hpy inside
	pypy/module/hpy_universal/test/_vendored with some temporary
	changes.

diff --git a/pypy/module/hpy_universal/test/__init__.py b/pypy/module/hpy_universal/test/__init__.py
new file mode 100644
diff --git a/pypy/module/hpy_universal/test/_vendored/__init__.py b/pypy/module/hpy_universal/test/_vendored/__init__.py
new file mode 100644
diff --git a/pypy/module/hpy_universal/test/_vendored/include/cpython/hpy.h b/pypy/module/hpy_universal/test/_vendored/include/cpython/hpy.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/test/_vendored/include/cpython/hpy.h
@@ -0,0 +1,142 @@
+#ifndef HPy_CPYTHON_H
+#define HPy_CPYTHON_H
+
+
+
+
+/* XXX: it would be nice if we could include hpy.h WITHOUT bringing in all the
+   stuff from Python.h, to make sure that people don't use the CPython API by
+   mistake. How to achieve it, though? */
+
+/* XXX: should we:
+ *    - enforce PY_SSIZE_T_CLEAN in hpy
+ *    - make it optional
+ *    - make it the default but give a way to disable it?
+ */
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+
+#ifdef __GNUC__
+#define HPyAPI_FUNC(restype)  __attribute__((unused)) static inline restype
+#else
+#define HPyAPI_FUNC(restype)  static inline restype
+#endif
+
+
+typedef struct { PyObject *_o; } HPy;
+typedef long HPyContext;
+
+HPyAPI_FUNC(HPyContext)
+_HPyGetContext(void) {
+    return 42;
+}
+
+/* For internal usage only. These should be #undef at the end of this header.
+   If you need to convert HPy to PyObject* and vice-versa, you should use the
+   official way to do it (not implemented yet :)
+*/
+#define _h2py(x) (x._o)
+#define _py2h(o) ((HPy){o})
+
+
+#define HPy_NULL ((HPy){NULL})
+#define HPy_IsNull(x) ((x)._o == NULL)
+
+HPyAPI_FUNC(HPy)
+HPyNone_Get(HPyContext ctx)
+{
+    Py_INCREF(Py_None);
+    return _py2h(Py_None);
+}
+
+HPyAPI_FUNC(HPy)
+HPy_Dup(HPyContext ctx, HPy handle)
+{
+    Py_XINCREF(_h2py(handle));
+    return handle;
+}
+
+HPyAPI_FUNC(void)
+HPy_Close(HPyContext ctx, HPy handle)
+{
+    Py_XDECREF(_h2py(handle));
+}
+
+/* moduleobject.h */
+typedef PyModuleDef HPyModuleDef;
+#define HPyModuleDef_HEAD_INIT PyModuleDef_HEAD_INIT
+
+HPyAPI_FUNC(HPy)
+HPyModule_Create(HPyContext ctx, HPyModuleDef *mdef) {
+    return _py2h(PyModule_Create(mdef));
+}
+
+#define HPy_MODINIT(modname)                                   \
+    static HPy init_##modname##_impl(HPyContext ctx);          \
+    PyMODINIT_FUNC                                             \
+    PyInit_##modname(void)                                     \
+    {                                                          \
+        return _h2py(init_##modname##_impl(_HPyGetContext())); \
+    }
+
+/* methodobject.h */
+typedef PyMethodDef HPyMethodDef;
+
+
+/* function declaration */
+
+#define HPy_FUNCTION(NAME)                                              \
+    static HPy NAME##_impl(HPyContext, HPy, HPy);                       \
+    static PyObject* NAME(PyObject *self, PyObject *args)               \
+    {                                                                   \
+        return _h2py(NAME##_impl(_HPyGetContext(), _py2h(self), _py2h(args)));\
+    }
+
+
+HPyAPI_FUNC(int)
+HPyArg_ParseTuple(HPyContext ctx, HPy args, const char *fmt, ...)
+{
+    va_list vl;
+    va_start(vl, fmt);
+    int res = PyArg_VaParse(_h2py(args), fmt, vl);
+    va_end(vl);
+    /* XXX incref all returned 'PyObject*' */
+    return res;
+}
+
+
+HPyAPI_FUNC(HPy)
+HPyLong_FromLong(HPyContext ctx, long v)
+{
+    return _py2h(PyLong_FromLong(v));
+}
+
+HPyAPI_FUNC(HPy)
+HPyNumber_Add(HPyContext ctx, HPy x, HPy y)
+{
+    return _py2h(PyNumber_Add(_h2py(x), _h2py(y)));
+}
+
+HPyAPI_FUNC(HPy)
+HPyUnicode_FromString(HPyContext ctx, const char *utf8)
+{
+    return _py2h(PyUnicode_FromString(utf8));
+}
+
+HPyAPI_FUNC(HPy)
+HPy_FromPyObject(HPyContext ctx, PyObject *obj)
+{
+    Py_XINCREF(obj);
+    return _py2h(obj);
+}
+
+HPyAPI_FUNC(PyObject *)
+HPy_AsPyObject(HPyContext ctx, HPy h)
+{
+    PyObject *result = _h2py(h);
+    Py_XINCREF(result);
+    return result;
+}
+
+
+#endif /* !HPy_CPYTHON_H */
diff --git a/pypy/module/hpy_universal/test/_vendored/include/hpy.h b/pypy/module/hpy_universal/test/_vendored/include/hpy.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/test/_vendored/include/hpy.h
@@ -0,0 +1,10 @@
+#ifndef HPy_H
+#define HPy_H
+
+#ifdef HPY_UNIVERSAL_ABI
+#    include "universal/hpy.h"
+#else
+#    include "cpython/hpy.h"
+#endif
+
+#endif /* HPy_H */
diff --git a/pypy/module/hpy_universal/test/_vendored/include/universal/autogen_ctx.h b/pypy/module/hpy_universal/test/_vendored/include/universal/autogen_ctx.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/test/_vendored/include/universal/autogen_ctx.h
@@ -0,0 +1,24 @@
+
+/*
+   DO NOT EDIT THIS FILE!
+
+   This file is automatically generated by tools/autogen.py from tools/public_api.h.
+   Run this to regenerate:
+       make autogen
+
+*/
+
+struct _HPyContext_s {
+    int ctx_version;
+    HPy (*ctx_Module_Create)(HPyContext ctx, HPyModuleDef *def);
+    HPy (*ctx_None_Get)(HPyContext ctx);
+    HPy (*ctx_Dup)(HPyContext ctx, HPy h);
+    void (*ctx_Close)(HPyContext ctx, HPy h);
+    HPy (*ctx_Long_FromLong)(HPyContext ctx, long value);
+    int (*ctx_Arg_ParseTuple)(HPyContext ctx, HPy args, const char *fmt, va_list _vl);
+    HPy (*ctx_Number_Add)(HPyContext ctx, HPy x, HPy y);
+    HPy (*ctx_Unicode_FromString)(HPyContext ctx, const char *utf8);
+    HPy (*ctx_FromPyObject)(HPyContext ctx, struct _object *obj);
+    struct _object *(*ctx_AsPyObject)(HPyContext ctx, HPy h);
+    struct _object *(*ctx_CallRealFunctionFromTrampoline)(HPyContext ctx, struct _object *self, struct _object *args, HPyCFunction func);
+};
diff --git a/pypy/module/hpy_universal/test/_vendored/include/universal/autogen_func.h b/pypy/module/hpy_universal/test/_vendored/include/universal/autogen_func.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/test/_vendored/include/universal/autogen_func.h
@@ -0,0 +1,54 @@
+
+/*
+   DO NOT EDIT THIS FILE!
+
+   This file is automatically generated by tools/autogen.py from tools/public_api.h.
+   Run this to regenerate:
+       make autogen
+
+*/
+
+static inline HPy HPyModule_Create(HPyContext ctx, HPyModuleDef *def) {
+     return ctx->ctx_Module_Create ( ctx, def ); 
+}
+
+static inline HPy HPyNone_Get(HPyContext ctx) {
+     return ctx->ctx_None_Get ( ctx ); 
+}
+
+static inline HPy HPy_Dup(HPyContext ctx, HPy h) {
+     return ctx->ctx_Dup ( ctx, h ); 
+}
+
+static inline void HPy_Close(HPyContext ctx, HPy h) {
+     ctx->ctx_Close ( ctx, h ); 
+}
+
+static inline HPy HPyLong_FromLong(HPyContext ctx, long value) {
+     return ctx->ctx_Long_FromLong ( ctx, value ); 
+}
+
+static inline int HPyArg_ParseTuple(HPyContext ctx, HPy args, const char *fmt, ...) {
+     va_list _vl; va_start(_vl, fmt); int _res = ctx->ctx_Arg_ParseTuple ( ctx, args, fmt, _vl ); va_end(_vl); return _res; 
+}
+
+static inline HPy HPyNumber_Add(HPyContext ctx, HPy x, HPy y) {
+     return ctx->ctx_Number_Add ( ctx, x, y ); 
+}
+
+static inline HPy HPyUnicode_FromString(HPyContext ctx, const char *utf8) {
+     return ctx->ctx_Unicode_FromString ( ctx, utf8 ); 
+}
+
+static inline HPy HPy_FromPyObject(HPyContext ctx, struct _object *obj) {
+     return ctx->ctx_FromPyObject ( ctx, obj ); 
+}
+
+static inline struct _object *HPy_AsPyObject(HPyContext ctx, HPy h) {
+     return ctx->ctx_AsPyObject ( ctx, h ); 
+}
+
+static inline struct _object *_HPy_CallRealFunctionFromTrampoline(HPyContext ctx, struct _object *self, struct _object *args, HPyCFunction func) {
+     return ctx->ctx_CallRealFunctionFromTrampoline ( ctx, self, args, func ); 
+}
+
diff --git a/pypy/module/hpy_universal/test/_vendored/include/universal/hpy.h b/pypy/module/hpy_universal/test/_vendored/include/universal/hpy.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/test/_vendored/include/universal/hpy.h
@@ -0,0 +1,79 @@
+#ifndef HPy_UNIVERSAL_H
+#define HPy_UNIVERSAL_H
+
+#include <stdlib.h>
+#include <stdint.h>
+#include <stdarg.h>
+
+typedef intptr_t HPy_ssize_t;
+typedef struct { HPy_ssize_t _i; } HPy;
+
+typedef struct _HPyContext_s *HPyContext;
+typedef HPy (*HPyCFunction)(HPyContext, HPy self, HPy args);
+struct _object;  /* that's PyObject inside CPython */
+typedef struct _object *(*_HPy_CPyCFunction)(struct _object *self,
+                                             struct _object *args);
+
+#define HPy_NULL ((HPy){0})
+#define HPy_IsNull(x) ((x)._i == 0)
+
+typedef void (*_HPyMethodPairFunc)(HPyCFunction *out_func,
+                                   _HPy_CPyCFunction *out_trampoline);
+
+typedef struct {
+    const char   *ml_name;   /* The name of the built-in function/method */
+    _HPyMethodPairFunc ml_meth;   /* see HPy_FUNCTION() */
+    int          ml_flags;   /* Combination of METH_xxx flags, which mostly
+                                describe the args expected by the C func */
+    const char   *ml_doc;    /* The __doc__ attribute, or NULL */
+} HPyMethodDef;
+
+
+#define HPyModuleDef_HEAD_INIT NULL
+
+typedef struct {
+    void *dummy; // this is needed because we put a comma after HPyModuleDef_HEAD_INIT :(
+    const char* m_name;
+    const char* m_doc;
+    HPy_ssize_t m_size;
+    HPyMethodDef *m_methods;
+} HPyModuleDef;
+
+#define HPy_MODINIT(modname)                                   \
+    HPyContext _ctx_for_trampolines;                           \
+    static HPy init_##modname##_impl(HPyContext ctx);          \
+    HPy HPyInit_##modname(HPyContext ctx)                      \
+    {                                                          \
+        _ctx_for_trampolines = ctx;                            \
+        return init_##modname##_impl(ctx);                     \
+    }
+
+#include "autogen_ctx.h"
+#include "autogen_func.h"
+
+extern HPyContext _ctx_for_trampolines;
+
+
+#define HPy_FUNCTION(fnname)                                                   \
+    static HPy fnname##_impl(HPyContext ctx, HPy self, HPy args);              \
+    static struct _object *                                                    \
+    fnname##_trampoline(struct _object *self, struct _object *args)            \
+    {                                                                          \
+        return _HPy_CallRealFunctionFromTrampoline(                            \
+            _ctx_for_trampolines, self, args, fnname##_impl);                  \
+    }                                                                          \
+    static void                                                                \
+    fnname(HPyCFunction *out_func, _HPy_CPyCFunction *out_trampoline)          \
+    {                                                                          \
+        *out_func = fnname##_impl;                                             \
+        *out_trampoline = fnname##_trampoline;                                 \
+    }
+
+#define METH_VARARGS  0x0001
+#define METH_KEYWORDS 0x0002
+/* METH_NOARGS and METH_O must not be combined with the flags above. */
+#define METH_NOARGS   0x0004
+#define METH_O        0x0008
+
+
+#endif /* HPy_UNIVERSAL_H */
diff --git a/pypy/module/hpy_universal/test/_vendored/support.py b/pypy/module/hpy_universal/test/_vendored/support.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/test/_vendored/support.py
@@ -0,0 +1,183 @@
+import os, sys
+import pytest
+import re
+#import importlib.util
+#from importlib.machinery import ExtensionFileLoader
+
+THIS_DIR = os.path.dirname(__file__)
+INCLUDE_DIR = os.path.join(THIS_DIR, '../hpy-api/include')
+
+
+r_marker_init = re.compile(r"\s*@INIT\s*$")
+r_marker_export = re.compile(r"\s*@EXPORT\s+(\w+)\s+(METH_\w+)\s*$")
+
+INIT_TEMPLATE = """
+static HPyMethodDef MyTestMethods[] = {
+    %(methods)s
+    {NULL, NULL, 0, NULL}
+};
+
+static HPyModuleDef moduledef = {
+    HPyModuleDef_HEAD_INIT,
+    .m_name = "%(name)s",
+    .m_doc = "some test for hpy",
+    .m_size = -1,
+    .m_methods = MyTestMethods
+};
+
+HPy_MODINIT(%(name)s)
+static HPy init_%(name)s_impl(HPyContext ctx)
+{
+    HPy m;
+    m = HPyModule_Create(ctx, &moduledef);
+    if (HPy_IsNull(m))
+        return HPy_NULL;
+    return m;
+}
+"""
+
+
+def expand_template(source_template, name):
+    method_table = []
+    expanded_lines = ['#include <hpy.h>']
+    for line in source_template.split('\n'):
+        match = r_marker_init.match(line)
+        if match:
+            exp = INIT_TEMPLATE % {
+                'methods': '\n    '.join(method_table),
+                'name': name}
+            method_table = None   # don't fill it any more
+            expanded_lines.append(exp)
+            continue
+
+        match = r_marker_export.match(line)
+        if match:
+            ml_name, ml_flags = match.group(1), match.group(2)
+            method_table.append('{"%s", %s, %s, NULL},' % (
+                ml_name, ml_name, ml_flags))
+            continue
+
+        expanded_lines.append(line)
+    return '\n'.join(expanded_lines)
+
+
+#class HPyLoader(ExtensionFileLoader):
+#    def create_module(self, spec):
+#        import hpy_universal
+#        return hpy_universal.load(spec.origin, "HPyInit_" + spec.name)
+
+class ExtensionCompiler:
+    def __init__(self, tmpdir, abimode):
+        self.tmpdir = tmpdir
+        self.abimode = abimode
+
+    def make_module(self, source_template, name):
+        universal_mode = self.abimode == 'universal'
+        source = expand_template(source_template, name)
+        filename = self.tmpdir.join(name + '.c')
+        filename.write(source)
+        #
+        ext = get_extension(str(filename), name, include_dirs=[INCLUDE_DIR],
+                            extra_compile_args=['-Wfatal-errors'])
+        so_filename = c_compile(str(self.tmpdir), ext, compiler_verbose=False,
+                                universal_mode=universal_mode)
+        #
+        if universal_mode:
+            loader = HPyLoader(name, so_filename)
+            spec = importlib.util.spec_from_loader(name, loader)
+        else:
+            spec = importlib.util.spec_from_file_location(name, so_filename)
+        module = importlib.util.module_from_spec(spec)
+        sys.modules[name] = module
+        spec.loader.exec_module(module)
+        return module
+
+
+ at pytest.mark.usefixtures('initargs')
+class HPyTest:
+    @pytest.fixture()
+    def initargs(self, compiler):
+        self.compiler = compiler
+
+    def make_module(self, source_template, name='mytest'):
+        return self.compiler.make_module(source_template, name)
+
+
+# the few functions below are copied and adapted from cffi/ffiplatform.py
+
+def get_extension(srcfilename, modname, sources=(), **kwds):
+    from distutils.core import Extension
+    allsources = [srcfilename]
+    for src in sources:
+        allsources.append(os.path.normpath(src))
+    return Extension(name=modname, sources=allsources, **kwds)
+
+def c_compile(tmpdir, ext, compiler_verbose=0, debug=None,
+              universal_mode=False):
+    """Compile a C extension module using distutils."""
+
+    saved_environ = os.environ.copy()
+    try:
+        outputfilename = _build(tmpdir, ext, compiler_verbose, debug,
+                                universal_mode)
+        outputfilename = os.path.abspath(outputfilename)
+    finally:
+        # workaround for a distutils bugs where some env vars can
+        # become longer and longer every time it is used
+        for key, value in saved_environ.items():
+            if os.environ.get(key) != value:
+                os.environ[key] = value
+    return outputfilename
+
+def _build(tmpdir, ext, compiler_verbose=0, debug=None, universal_mode=False):
+    # XXX compact but horrible :-(
+    from distutils.core import Distribution
+    import distutils.errors, distutils.log
+    #
+    dist = Distribution({'ext_modules': [ext]})
+    dist.parse_config_files()
+    options = dist.get_option_dict('build_ext')
+    if debug is None:
+        debug = sys.flags.debug
+    options['debug'] = ('ffiplatform', debug)
+    options['force'] = ('ffiplatform', True)
+    options['build_lib'] = ('ffiplatform', tmpdir)
+    options['build_temp'] = ('ffiplatform', tmpdir)
+    #
+    old_level = distutils.log.set_threshold(0) or 0
+    try:
+        distutils.log.set_verbosity(compiler_verbose)
+        if universal_mode:
+            cmd_obj = dist.get_command_obj('build_ext')
+            cmd_obj.finalize_options()
+            soname = _build_universal(tmpdir, ext, cmd_obj.include_dirs)
+        else:
+            dist.run_command('build_ext')
+            cmd_obj = dist.get_command_obj('build_ext')
+            [soname] = cmd_obj.get_outputs()
+    finally:
+        distutils.log.set_threshold(old_level)
+    #
+    return soname
+
+def _build_universal(tmpdir, ext, cpython_include_dirs):
+    from distutils.ccompiler import new_compiler, get_default_compiler
+    from distutils.sysconfig import customize_compiler
+
+    compiler = new_compiler(get_default_compiler())
+    customize_compiler(compiler)
+
+    include_dirs = ext.include_dirs + cpython_include_dirs
+    objects = compiler.compile(ext.sources,
+                               output_dir=tmpdir,
+                               macros=[('HPY_UNIVERSAL_ABI', None)],
+                               include_dirs=include_dirs)
+
+    filename = ext.name + '.hpy.so'
+    compiler.link(compiler.SHARED_LIBRARY,
+                  objects,
+                  filename,
+                  tmpdir
+                  # export_symbols=...
+                  )
+    return os.path.join(tmpdir, filename)
diff --git a/pypy/module/hpy_universal/test/support.py b/pypy/module/hpy_universal/test/support.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/test/support.py
@@ -0,0 +1,49 @@
+import py
+import pytest
+from pypy.interpreter.gateway import interp2app, unwrap_spec
+
+from rpython.tool.udir import udir
+from ._vendored import support as _support
+
+INCLUDE_DIR = str(py.path.local(__file__).dirpath().join('_vendored/include'))
+
+class ExtensionCompiler(object):
+    def __init__(self, base_dir):
+        self.base_dir = base_dir
+
+    def get_builddir(self, name='mytest'):
+        builddir = py.path.local.make_numbered_dir(
+            rootdir=py.path.local(self.base_dir),
+            prefix=name + '-',
+            keep=0)  # keep everything
+        return builddir
+
+
+class HPyTest(object):
+    def setup_class(cls):
+        if cls.runappdirect:
+            pytest.skip()
+        cls.compiler = ExtensionCompiler(udir)
+
+        @unwrap_spec(source_template='text', name='text')
+        def descr_make_module(space, source_template, name='mytest'):
+            source = _support.expand_template(source_template, name)
+            tmpdir = cls.compiler.get_builddir()
+            filename = tmpdir.join(name+ '.c')
+            filename.write(source)
+            #
+            ext = _support.get_extension(str(filename), name, include_dirs=[INCLUDE_DIR],
+                                extra_compile_args=['-Wfatal-errors'])
+            so_filename = _support.c_compile(str(tmpdir), ext, compiler_verbose=False,
+                                    universal_mode=True)
+            #
+            w_mod = space.appexec(
+                [space.newtext(so_filename), space.newtext(name)],
+                """(path, modname):
+                    from hpy_universal import load
+                    return load(path, 'HPyInit_' + modname)
+                """
+            )
+            return w_mod
+        cls.w_make_module = cls.space.wrap(interp2app(descr_make_module))
+
diff --git a/pypy/module/hpy_universal/test/test_basic.py b/pypy/module/hpy_universal/test/test_basic.py
--- a/pypy/module/hpy_universal/test/test_basic.py
+++ b/pypy/module/hpy_universal/test/test_basic.py
@@ -1,4 +1,17 @@
-class AppTestBasic(object):
+from .support import HPyTest
+
+class AppTestBasic(HPyTest):
     spaceconfig = {'usemodules': ['hpy_universal']}
     def test_import(self):
         import hpy_universal
+
+    def test_empty_module(self):
+        import sys
+        mod = self.make_module("""
+            @INIT
+        """)
+        assert type(mod) is type(sys)
+        assert mod.__loader__.name == 'mytest'
+        assert mod.__spec__.loader is mod.__loader__
+        assert mod.__file__
+

From pypy.commits at gmail.com  Fri Nov 15 15:01:05 2019
From: pypy.commits at gmail.com (arigo)
Date: Fri, 15 Nov 2019 12:01:05 -0800 (PST)
Subject: [pypy-commit] pypy default: Issue #3114
Message-ID: <5dcf0401.1c69fb81.c4469.bffd@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r98060:4234613045bb
Date: 2019-11-15 20:59 +0100
http://bitbucket.org/pypy/pypy/changeset/4234613045bb/

Log:	Issue #3114

	Allow general buffers in socket.setsockopt(), like CPython does

diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -458,7 +458,7 @@
         except OperationError as e:
             if e.async(space):
                 raise
-            optval = space.bytes_w(w_optval)
+            optval = space.bufferstr_w(w_optval)
             try:
                 self.sock.setsockopt(level, optname, optval)
             except SocketError as e:
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -567,6 +567,13 @@
                                 intsize)
         (reuse,) = struct.unpack('i', reusestr)
         assert reuse != 0
+        # try to call setsockopt() with a buffer argument
+        reusestr = struct.pack('i', 0)
+        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, buffer(reusestr))
+        reusestr = s.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR,
+                                intsize)
+        (reuse,) = struct.unpack('i', reusestr)
+        assert reuse == 0
 
     def test_getsetsockopt_zero(self):
         # related to issue #2561: when specifying the buffer size param:

From pypy.commits at gmail.com  Sat Nov 16 05:24:10 2019
From: pypy.commits at gmail.com (arigo)
Date: Sat, 16 Nov 2019 02:24:10 -0800 (PST)
Subject: [pypy-commit] pypy default: oops, aarch64 fix
Message-ID: <5dcfce4a.1c69fb81.eaf73.ac27@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r98063:41990e65c077
Date: 2019-11-16 11:23 +0100
http://bitbucket.org/pypy/pypy/changeset/41990e65c077/

Log:	oops, aarch64 fix

diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -267,7 +267,8 @@
 if sys.platform == 'win32':
     DEFAULT_SFLAGS_PLATFORM = SF_MSVC_BITFIELDS
 else:
-    if rffi_platform.getdefined('__arm__', ''):
+    if (rffi_platform.getdefined('__arm__', '') or
+        rffi_platform.getdefined('__aarch64__', '')):
         DEFAULT_SFLAGS_PLATFORM = SF_GCC_ARM_BITFIELDS
     else:
         DEFAULT_SFLAGS_PLATFORM = SF_GCC_X86_BITFIELDS

From pypy.commits at gmail.com  Sat Nov 16 06:52:27 2019
From: pypy.commits at gmail.com (arigo)
Date: Sat, 16 Nov 2019 03:52:27 -0800 (PST)
Subject: [pypy-commit] pypy int-test-is-zero: Generate more compact code for
 bit tests using the TEST instruction:
Message-ID: <5dcfe2fb.1c69fb81.7fd28.ada5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: int-test-is-zero
Changeset: r98064:092643ba66ac
Date: 2019-11-16 12:51 +0100
http://bitbucket.org/pypy/pypy/changeset/092643ba66ac/

Log:	Generate more compact code for bit tests using the TEST instruction:

	 i1 = int_and(i0, 7); i2 = int_is_zero(i1); guard_true(i2)

	 =>

	 i2 = int_test_is_zero(i0, 7); guard_true(i2)

	The more compact version doesn't need to allocate a register, copy a
	value there, AND a constant, and then CMP the result with zero.
	Instead a single TEST instruction suffices.

diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -1491,6 +1491,8 @@
             (rop.INT_NE, lambda x, y: x != y),
             (rop.INT_GT, lambda x, y: x > y),
             (rop.INT_GE, lambda x, y: x >= y),
+            (rop.INT_TEST_IS_ZERO, lambda x, y: (x & y) == 0),
+            (rop.INT_TEST_IS_TRUE, lambda x, y: (x & y) != 0),
             ]:
             for opguard, guard_case in [
                 (rop.GUARD_FALSE, False),
diff --git a/rpython/jit/backend/test/test_random.py b/rpython/jit/backend/test/test_random.py
--- a/rpython/jit/backend/test/test_random.py
+++ b/rpython/jit/backend/test/test_random.py
@@ -546,6 +546,8 @@
             rop.UINT_LE,
             rop.UINT_GT,
             rop.UINT_GE,
+            rop.INT_TEST_IS_ZERO,
+            rop.INT_TEST_IS_TRUE,
             ]:
     OPERATIONS.append(BinaryOperation(_op, boolres=True))
 
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1281,6 +1281,17 @@
                 self.flush_cc(cond, result_loc)
         return genop_cmp
 
+    def _testop(cond):
+        cond = rx86.Conditions[cond]
+        #
+        def genop_test(self, op, arglocs, result_loc):
+            if arglocs[1].is_stack() or isinstance(arglocs[0], ImmedLoc):
+                self.mc.TEST(arglocs[1], arglocs[0])
+            else:
+                self.mc.TEST(arglocs[0], arglocs[1])
+            self.flush_cc(cond, result_loc)
+        return genop_test
+
     def _if_parity_clear_zero_and_carry(self):
         jnp_location = self.mc.emit_forward_jump('NP')
         # CMP EBP, 0: as EBP cannot be null here, that operation should
@@ -1401,6 +1412,9 @@
     genop_float_gt = _cmpop_float("A", "B")
     genop_float_ge = _cmpop_float("AE","BE")
 
+    genop_int_test_is_zero = _testop("Z")
+    genop_int_test_is_true = _testop("NZ")
+
     def genop_math_sqrt(self, op, arglocs, resloc):
         self.mc.SQRTSD(arglocs[0], resloc)
 
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -644,6 +644,8 @@
     consider_uint_ge = _consider_compop
     consider_ptr_eq = consider_instance_ptr_eq = _consider_compop
     consider_ptr_ne = consider_instance_ptr_ne = _consider_compop
+    consider_int_test_is_zero = _consider_compop
+    consider_int_test_is_true = _consider_compop
 
     def _consider_float_op(self, op):
         loc1 = self.xrm.loc(op.getarg(1))
diff --git a/rpython/jit/backend/x86/rx86.py b/rpython/jit/backend/x86/rx86.py
--- a/rpython/jit/backend/x86/rx86.py
+++ b/rpython/jit/backend/x86/rx86.py
@@ -699,6 +699,9 @@
     TEST_ai = insn(rex_w, '\xF7', orbyte(0<<3), mem_reg_plus_scaled_reg_plus_const(1), immediate(2))
     TEST_mi = insn(rex_w, '\xF7', orbyte(0<<3), mem_reg_plus_const(1), immediate(2))
     TEST_ji = insn(rex_w, '\xF7', orbyte(0<<3), abs_(1), immediate(2))
+    TEST_ri = insn(rex_w, '\xF7', orbyte(0<<3), register(1), '\xC0', immediate(2))
+    TEST_bi = insn(rex_w, '\xF7', orbyte(0<<3), stack_bp(1), immediate(2))
+    TEST_br = insn(rex_w, '\x85', register(2,8), stack_bp(1))
 
     BTS_mr = insn(rex_w, '\x0F\xAB', register(2,8), mem_reg_plus_const(1))
     BTS_jr = insn(rex_w, '\x0F\xAB', register(2,8), abs_(1))
diff --git a/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py b/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py
--- a/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py
+++ b/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py
@@ -311,6 +311,8 @@
                 return []   # MOV AL, [immediate]: there is a special encoding
             if methname == 'MOV8_jr' and args[1] == rx86.R.al:
                 return []   # MOV [immediate], AL: there is a special encoding
+            if methname == 'TEST_ri' and args[0] == rx86.R.eax:
+                return []  # TEST EAX, constant: there is a special encoding
 
             return [args]
 
diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -542,6 +542,12 @@
     @arguments("i", "i", returns="i")
     def bhimpl_int_signext(a, b):
         return int_signext(a, b)
+    @arguments("i", "i", returns="i")
+    def bhimpl_int_test_is_zero(a, b):
+        return (a & b) == 0
+    @arguments("i", "i", returns="i")
+    def bhimpl_int_test_is_true(a, b):
+        return (a & b) != 0
 
     @arguments("i", "i", returns="i")
     def bhimpl_uint_lt(a, b):
diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -442,6 +442,8 @@
                          rop.GC_STORE_INDEXED,
                          rop.LOAD_FROM_GC_TABLE,
                          rop.LOAD_EFFECTIVE_ADDRESS,
+                         rop.INT_TEST_IS_ZERO,
+                         rop.INT_TEST_IS_TRUE,
                          ):      # list of opcodes never executed by pyjitpl
                 continue
             if rop._VEC_PURE_FIRST <= value <= rop._VEC_PURE_LAST:
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -646,6 +646,15 @@
         elif info == INFO_NULL:
             self.make_constant_int(op, not expect_nonnull)
         else:
+            box = get_box_replacement(box)
+            box1 = self.optimizer.as_operation(box)
+            if box1 is not None and box1.getopnum() == rop.INT_AND:
+                if expect_nonnull:
+                    opnum = rop.INT_TEST_IS_TRUE
+                else:
+                    opnum = rop.INT_TEST_IS_ZERO
+                args = [box1.getarg(0), box1.getarg(1)]
+                op = self.replace_op_with(op, opnum, args=args)
             return self.emit(op)
 
     def optimize_INT_IS_TRUE(self, op):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -5790,3 +5790,33 @@
         i57 = int_or(i51, i52)
         """
         self.optimize_loop(ops, expected)
+
+    def test_int_test_is_zero(self):
+        ops = """
+        [i1, i2]
+        i51 = int_and(i1, i2)
+        i52 = int_is_zero(i51)
+        guard_true(i52) []
+        """
+        expected = """
+        [i1, i2]
+        i51 = int_and(i1, i2)      # likely dead instruction
+        i52 = int_test_is_zero(i1, i2)
+        guard_true(i52) []
+        """
+        self.optimize_loop(ops, expected)
+
+    def test_int_test_is_true(self):
+        ops = """
+        [i1, i2]
+        i51 = int_and(i1, i2)
+        i52 = int_is_true(i51)
+        guard_true(i52) []
+        """
+        expected = """
+        [i1, i2]
+        i51 = int_and(i1, i2)      # likely dead instruction
+        i52 = int_test_is_true(i1, i2)
+        guard_true(i52) []
+        """
+        self.optimize_loop(ops, expected)
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -1036,6 +1036,8 @@
     'INT_NEG/1/i',
     'INT_INVERT/1/i',
     'INT_FORCE_GE_ZERO/1/i',
+    'INT_TEST_IS_ZERO/2b/i',
+    'INT_TEST_IS_TRUE/2b/i',
     #
     'SAME_AS/1/ifr',      # gets a Const or a Box, turns it into another Box
     'CAST_PTR_TO_INT/1/i',
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4832,3 +4832,11 @@
         res2 = self.interp_operations(f, [6])
         assert res1 == res2
         self.check_operations_history(guard_class=1, record_exact_class=0)
+
+    def test_int_test_instructions(self):
+        def f(x, y):
+            if (x & 7) == 0 and (y & 7) != 0:
+                return 1
+            return 0
+        res = self.interp_operations(f, [24, 25])
+        assert res == 1
diff --git a/rpython/jit/metainterp/test/test_executor.py b/rpython/jit/metainterp/test/test_executor.py
--- a/rpython/jit/metainterp/test/test_executor.py
+++ b/rpython/jit/metainterp/test/test_executor.py
@@ -187,6 +187,8 @@
         (rop.UINT_LE, lambda x, y: r_uint(x) <= r_uint(y)),
         (rop.UINT_GT, lambda x, y: r_uint(x) >  r_uint(y)),
         (rop.UINT_GE, lambda x, y: r_uint(x) >= r_uint(y)),
+        (rop.INT_TEST_IS_ZERO, lambda x, y: (x & y) == 0),
+        (rop.INT_TEST_IS_TRUE, lambda x, y: (x & y) != 0),
         ]:
         for i in range(20):
             x = pick()

From pypy.commits at gmail.com  Sat Nov 16 07:44:49 2019
From: pypy.commits at gmail.com (arigo)
Date: Sat, 16 Nov 2019 04:44:49 -0800 (PST)
Subject: [pypy-commit] pypy int-test-is-zero: Make the usage of INT_TEST_xxx
 conditional on the backend
Message-ID: <5dcfef41.1c69fb81.edf9f.09ca@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: int-test-is-zero
Changeset: r98065:56ea3adf27c4
Date: 2019-11-16 13:44 +0100
http://bitbucket.org/pypy/pypy/changeset/56ea3adf27c4/

Log:	Make the usage of INT_TEST_xxx conditional on the backend

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -333,6 +333,7 @@
     vector_ext.enable(16, accum=True)
     vector_ext.setup_once = lambda asm: asm
     load_supported_factors = (1,2,4,8)
+    supports_int_test_instructions = True
     assembler = None
 
     def __init__(self, rtyper, stats=None, *ignored_args, **kwds):
diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py
--- a/rpython/jit/backend/model.py
+++ b/rpython/jit/backend/model.py
@@ -20,6 +20,7 @@
     supports_singlefloats = False
     supports_guard_gc_type = False
     supports_load_effective_address = False
+    supports_int_test_instructions = False
 
     propagate_exception_descr = None
 
diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py
--- a/rpython/jit/backend/x86/runner.py
+++ b/rpython/jit/backend/x86/runner.py
@@ -17,6 +17,7 @@
     supports_floats = True
     supports_singlefloats = True
     supports_load_effective_address = True
+    supports_int_test_instructions = True
 
     dont_keepalive_stuff = False # for tests
     with_threads = False
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -646,15 +646,16 @@
         elif info == INFO_NULL:
             self.make_constant_int(op, not expect_nonnull)
         else:
-            box = get_box_replacement(box)
-            box1 = self.optimizer.as_operation(box)
-            if box1 is not None and box1.getopnum() == rop.INT_AND:
-                if expect_nonnull:
-                    opnum = rop.INT_TEST_IS_TRUE
-                else:
-                    opnum = rop.INT_TEST_IS_ZERO
-                args = [box1.getarg(0), box1.getarg(1)]
-                op = self.replace_op_with(op, opnum, args=args)
+            if self.optimizer.cpu.supports_int_test_instructions:
+                box = get_box_replacement(box)
+                box1 = self.optimizer.as_operation(box)
+                if box1 is not None and box1.getopnum() == rop.INT_AND:
+                    if expect_nonnull:
+                        opnum = rop.INT_TEST_IS_TRUE
+                    else:
+                        opnum = rop.INT_TEST_IS_ZERO
+                    args = [box1.getarg(0), box1.getarg(1)]
+                    op = self.replace_op_with(op, opnum, args=args)
             return self.emit(op)
 
     def optimize_INT_IS_TRUE(self, op):

From pypy.commits at gmail.com  Sat Nov 16 08:03:51 2019
From: pypy.commits at gmail.com (arigo)
Date: Sat, 16 Nov 2019 05:03:51 -0800 (PST)
Subject: [pypy-commit] pypy int-test-is-zero: Add a test_pypy_c test for
 int_test_is_true
Message-ID: <5dcff3b7.1c69fb81.b5677.50fe@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: int-test-is-zero
Changeset: r98066:abbcc2ad6b7f
Date: 2019-11-16 13:00 +0000
http://bitbucket.org/pypy/pypy/changeset/abbcc2ad6b7f/

Log:	Add a test_pypy_c test for int_test_is_true

diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -420,3 +420,20 @@
         # the following assertion fails if the loop was cancelled due
         # to "abort: vable escape"
         assert len(loops) == 1
+
+    def test_bit_check(self):
+        def main(n):
+            x = 0
+            while n:
+                y = bool(n & 7)    # ID: bitcheck
+                x += y
+                n -= 1
+
+        log = self.run(main, [300])
+        loop, = log.loops_by_id("bitcheck")
+        assert loop.match_by_id("bitcheck", """
+            guard_not_invalidated?
+            i11 = int_and(i7, 7)    # not used
+            i12 = int_test_is_true(i7, 7)
+            guard_true(i12, descr=...)
+        """)

From pypy.commits at gmail.com  Sat Nov 16 08:03:53 2019
From: pypy.commits at gmail.com (arigo)
Date: Sat, 16 Nov 2019 05:03:53 -0800 (PST)
Subject: [pypy-commit] pypy int-test-is-zero: merge heads
Message-ID: <5dcff3b9.1c69fb81.6633c.a1ca@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: int-test-is-zero
Changeset: r98067:f2afd59f9872
Date: 2019-11-16 14:02 +0100
http://bitbucket.org/pypy/pypy/changeset/f2afd59f9872/

Log:	merge heads

diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -420,3 +420,20 @@
         # the following assertion fails if the loop was cancelled due
         # to "abort: vable escape"
         assert len(loops) == 1
+
+    def test_bit_check(self):
+        def main(n):
+            x = 0
+            while n:
+                y = bool(n & 7)    # ID: bitcheck
+                x += y
+                n -= 1
+
+        log = self.run(main, [300])
+        loop, = log.loops_by_id("bitcheck")
+        assert loop.match_by_id("bitcheck", """
+            guard_not_invalidated?
+            i11 = int_and(i7, 7)    # not used
+            i12 = int_test_is_true(i7, 7)
+            guard_true(i12, descr=...)
+        """)

From pypy.commits at gmail.com  Sat Nov 16 08:11:26 2019
From: pypy.commits at gmail.com (arigo)
Date: Sat, 16 Nov 2019 05:11:26 -0800 (PST)
Subject: [pypy-commit] pypy int-test-is-zero: skip this test on non-x86
 platforms
Message-ID: <5dcff57e.1c69fb81.9dc83.b56f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: int-test-is-zero
Changeset: r98068:bbc157fc2398
Date: 2019-11-16 14:09 +0100
http://bitbucket.org/pypy/pypy/changeset/bbc157fc2398/

Log:	skip this test on non-x86 platforms

diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -1,4 +1,5 @@
 import py, sys
+import platform
 from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC
 
 
@@ -422,6 +423,9 @@
         assert len(loops) == 1
 
     def test_bit_check(self):
+        if not platform.machine().startswith('x86'):
+            py.test.skip("only x86 supports int_test_instructions for now")
+
         def main(n):
             x = 0
             while n:

From pypy.commits at gmail.com  Sat Nov 16 08:13:40 2019
From: pypy.commits at gmail.com (arigo)
Date: Sat, 16 Nov 2019 05:13:40 -0800 (PST)
Subject: [pypy-commit] pypy int-test-is-zero: Close branch int-test-is-zero
Message-ID: <5dcff604.1c69fb81.530e.8268@mx.google.com>

Author: Armin Rigo <armin.rigo at gmail.com>
Branch: int-test-is-zero
Changeset: r98069:d731a41aae59
Date: 2019-11-16 13:13 +0000
http://bitbucket.org/pypy/pypy/changeset/d731a41aae59/

Log:	Close branch int-test-is-zero


From pypy.commits at gmail.com  Sat Nov 16 08:13:58 2019
From: pypy.commits at gmail.com (arigo)
Date: Sat, 16 Nov 2019 05:13:58 -0800 (PST)
Subject: [pypy-commit] pypy default: Merged in int-test-is-zero (pull
 request #684)
Message-ID: <5dcff616.1c69fb81.c559a.21f9@mx.google.com>

Author: Armin Rigo <armin.rigo at gmail.com>
Branch: 
Changeset: r98070:18443d3a74d5
Date: 2019-11-16 13:13 +0000
http://bitbucket.org/pypy/pypy/changeset/18443d3a74d5/

Log:	Merged in int-test-is-zero (pull request #684)

	Use the TEST assembler instruction on x86

diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -1,4 +1,5 @@
 import py, sys
+import platform
 from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC
 
 
@@ -420,3 +421,23 @@
         # the following assertion fails if the loop was cancelled due
         # to "abort: vable escape"
         assert len(loops) == 1
+
+    def test_bit_check(self):
+        if not platform.machine().startswith('x86'):
+            py.test.skip("only x86 supports int_test_instructions for now")
+
+        def main(n):
+            x = 0
+            while n:
+                y = bool(n & 7)    # ID: bitcheck
+                x += y
+                n -= 1
+
+        log = self.run(main, [300])
+        loop, = log.loops_by_id("bitcheck")
+        assert loop.match_by_id("bitcheck", """
+            guard_not_invalidated?
+            i11 = int_and(i7, 7)    # not used
+            i12 = int_test_is_true(i7, 7)
+            guard_true(i12, descr=...)
+        """)
diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -333,6 +333,7 @@
     vector_ext.enable(16, accum=True)
     vector_ext.setup_once = lambda asm: asm
     load_supported_factors = (1,2,4,8)
+    supports_int_test_instructions = True
     assembler = None
 
     def __init__(self, rtyper, stats=None, *ignored_args, **kwds):
diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py
--- a/rpython/jit/backend/model.py
+++ b/rpython/jit/backend/model.py
@@ -20,6 +20,7 @@
     supports_singlefloats = False
     supports_guard_gc_type = False
     supports_load_effective_address = False
+    supports_int_test_instructions = False
 
     propagate_exception_descr = None
 
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -1491,6 +1491,8 @@
             (rop.INT_NE, lambda x, y: x != y),
             (rop.INT_GT, lambda x, y: x > y),
             (rop.INT_GE, lambda x, y: x >= y),
+            (rop.INT_TEST_IS_ZERO, lambda x, y: (x & y) == 0),
+            (rop.INT_TEST_IS_TRUE, lambda x, y: (x & y) != 0),
             ]:
             for opguard, guard_case in [
                 (rop.GUARD_FALSE, False),
diff --git a/rpython/jit/backend/test/test_random.py b/rpython/jit/backend/test/test_random.py
--- a/rpython/jit/backend/test/test_random.py
+++ b/rpython/jit/backend/test/test_random.py
@@ -546,6 +546,8 @@
             rop.UINT_LE,
             rop.UINT_GT,
             rop.UINT_GE,
+            rop.INT_TEST_IS_ZERO,
+            rop.INT_TEST_IS_TRUE,
             ]:
     OPERATIONS.append(BinaryOperation(_op, boolres=True))
 
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1281,6 +1281,17 @@
                 self.flush_cc(cond, result_loc)
         return genop_cmp
 
+    def _testop(cond):
+        cond = rx86.Conditions[cond]
+        #
+        def genop_test(self, op, arglocs, result_loc):
+            if arglocs[1].is_stack() or isinstance(arglocs[0], ImmedLoc):
+                self.mc.TEST(arglocs[1], arglocs[0])
+            else:
+                self.mc.TEST(arglocs[0], arglocs[1])
+            self.flush_cc(cond, result_loc)
+        return genop_test
+
     def _if_parity_clear_zero_and_carry(self):
         jnp_location = self.mc.emit_forward_jump('NP')
         # CMP EBP, 0: as EBP cannot be null here, that operation should
@@ -1401,6 +1412,9 @@
     genop_float_gt = _cmpop_float("A", "B")
     genop_float_ge = _cmpop_float("AE","BE")
 
+    genop_int_test_is_zero = _testop("Z")
+    genop_int_test_is_true = _testop("NZ")
+
     def genop_math_sqrt(self, op, arglocs, resloc):
         self.mc.SQRTSD(arglocs[0], resloc)
 
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -644,6 +644,8 @@
     consider_uint_ge = _consider_compop
     consider_ptr_eq = consider_instance_ptr_eq = _consider_compop
     consider_ptr_ne = consider_instance_ptr_ne = _consider_compop
+    consider_int_test_is_zero = _consider_compop
+    consider_int_test_is_true = _consider_compop
 
     def _consider_float_op(self, op):
         loc1 = self.xrm.loc(op.getarg(1))
diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py
--- a/rpython/jit/backend/x86/runner.py
+++ b/rpython/jit/backend/x86/runner.py
@@ -17,6 +17,7 @@
     supports_floats = True
     supports_singlefloats = True
     supports_load_effective_address = True
+    supports_int_test_instructions = True
 
     dont_keepalive_stuff = False # for tests
     with_threads = False
diff --git a/rpython/jit/backend/x86/rx86.py b/rpython/jit/backend/x86/rx86.py
--- a/rpython/jit/backend/x86/rx86.py
+++ b/rpython/jit/backend/x86/rx86.py
@@ -699,6 +699,9 @@
     TEST_ai = insn(rex_w, '\xF7', orbyte(0<<3), mem_reg_plus_scaled_reg_plus_const(1), immediate(2))
     TEST_mi = insn(rex_w, '\xF7', orbyte(0<<3), mem_reg_plus_const(1), immediate(2))
     TEST_ji = insn(rex_w, '\xF7', orbyte(0<<3), abs_(1), immediate(2))
+    TEST_ri = insn(rex_w, '\xF7', orbyte(0<<3), register(1), '\xC0', immediate(2))
+    TEST_bi = insn(rex_w, '\xF7', orbyte(0<<3), stack_bp(1), immediate(2))
+    TEST_br = insn(rex_w, '\x85', register(2,8), stack_bp(1))
 
     BTS_mr = insn(rex_w, '\x0F\xAB', register(2,8), mem_reg_plus_const(1))
     BTS_jr = insn(rex_w, '\x0F\xAB', register(2,8), abs_(1))
diff --git a/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py b/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py
--- a/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py
+++ b/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py
@@ -311,6 +311,8 @@
                 return []   # MOV AL, [immediate]: there is a special encoding
             if methname == 'MOV8_jr' and args[1] == rx86.R.al:
                 return []   # MOV [immediate], AL: there is a special encoding
+            if methname == 'TEST_ri' and args[0] == rx86.R.eax:
+                return []  # TEST EAX, constant: there is a special encoding
 
             return [args]
 
diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -542,6 +542,12 @@
     @arguments("i", "i", returns="i")
     def bhimpl_int_signext(a, b):
         return int_signext(a, b)
+    @arguments("i", "i", returns="i")
+    def bhimpl_int_test_is_zero(a, b):
+        return (a & b) == 0
+    @arguments("i", "i", returns="i")
+    def bhimpl_int_test_is_true(a, b):
+        return (a & b) != 0
 
     @arguments("i", "i", returns="i")
     def bhimpl_uint_lt(a, b):
diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -442,6 +442,8 @@
                          rop.GC_STORE_INDEXED,
                          rop.LOAD_FROM_GC_TABLE,
                          rop.LOAD_EFFECTIVE_ADDRESS,
+                         rop.INT_TEST_IS_ZERO,
+                         rop.INT_TEST_IS_TRUE,
                          ):      # list of opcodes never executed by pyjitpl
                 continue
             if rop._VEC_PURE_FIRST <= value <= rop._VEC_PURE_LAST:
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -646,6 +646,16 @@
         elif info == INFO_NULL:
             self.make_constant_int(op, not expect_nonnull)
         else:
+            if self.optimizer.cpu.supports_int_test_instructions:
+                box = get_box_replacement(box)
+                box1 = self.optimizer.as_operation(box)
+                if box1 is not None and box1.getopnum() == rop.INT_AND:
+                    if expect_nonnull:
+                        opnum = rop.INT_TEST_IS_TRUE
+                    else:
+                        opnum = rop.INT_TEST_IS_ZERO
+                    args = [box1.getarg(0), box1.getarg(1)]
+                    op = self.replace_op_with(op, opnum, args=args)
             return self.emit(op)
 
     def optimize_INT_IS_TRUE(self, op):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -5790,3 +5790,33 @@
         i57 = int_or(i51, i52)
         """
         self.optimize_loop(ops, expected)
+
+    def test_int_test_is_zero(self):
+        ops = """
+        [i1, i2]
+        i51 = int_and(i1, i2)
+        i52 = int_is_zero(i51)
+        guard_true(i52) []
+        """
+        expected = """
+        [i1, i2]
+        i51 = int_and(i1, i2)      # likely dead instruction
+        i52 = int_test_is_zero(i1, i2)
+        guard_true(i52) []
+        """
+        self.optimize_loop(ops, expected)
+
+    def test_int_test_is_true(self):
+        ops = """
+        [i1, i2]
+        i51 = int_and(i1, i2)
+        i52 = int_is_true(i51)
+        guard_true(i52) []
+        """
+        expected = """
+        [i1, i2]
+        i51 = int_and(i1, i2)      # likely dead instruction
+        i52 = int_test_is_true(i1, i2)
+        guard_true(i52) []
+        """
+        self.optimize_loop(ops, expected)
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -1036,6 +1036,8 @@
     'INT_NEG/1/i',
     'INT_INVERT/1/i',
     'INT_FORCE_GE_ZERO/1/i',
+    'INT_TEST_IS_ZERO/2b/i',
+    'INT_TEST_IS_TRUE/2b/i',
     #
     'SAME_AS/1/ifr',      # gets a Const or a Box, turns it into another Box
     'CAST_PTR_TO_INT/1/i',
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4832,3 +4832,11 @@
         res2 = self.interp_operations(f, [6])
         assert res1 == res2
         self.check_operations_history(guard_class=1, record_exact_class=0)
+
+    def test_int_test_instructions(self):
+        def f(x, y):
+            if (x & 7) == 0 and (y & 7) != 0:
+                return 1
+            return 0
+        res = self.interp_operations(f, [24, 25])
+        assert res == 1
diff --git a/rpython/jit/metainterp/test/test_executor.py b/rpython/jit/metainterp/test/test_executor.py
--- a/rpython/jit/metainterp/test/test_executor.py
+++ b/rpython/jit/metainterp/test/test_executor.py
@@ -187,6 +187,8 @@
         (rop.UINT_LE, lambda x, y: r_uint(x) <= r_uint(y)),
         (rop.UINT_GT, lambda x, y: r_uint(x) >  r_uint(y)),
         (rop.UINT_GE, lambda x, y: r_uint(x) >= r_uint(y)),
+        (rop.INT_TEST_IS_ZERO, lambda x, y: (x & y) == 0),
+        (rop.INT_TEST_IS_TRUE, lambda x, y: (x & y) != 0),
         ]:
         for i in range(20):
             x = pick()

From pypy.commits at gmail.com  Sat Nov 16 08:15:09 2019
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 16 Nov 2019 05:15:09 -0800 (PST)
Subject: [pypy-commit] pypy hpy: Fix cparser issue with struct types
 appearing in function arguments
Message-ID: <5dcff65d.1c69fb81.4ed54.aff5@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: hpy
Changeset: r98071:e2a687fd8ab3
Date: 2019-11-16 14:13 +0100
http://bitbucket.org/pypy/pypy/changeset/e2a687fd8ab3/

Log:	Fix cparser issue with struct types appearing in function arguments

diff --git a/pypy/module/cpyext/cparser.py b/pypy/module/cpyext/cparser.py
--- a/pypy/module/cpyext/cparser.py
+++ b/pypy/module/cpyext/cparser.py
@@ -732,6 +732,7 @@
             if tp.type_name is None:
                 tp.type_name = name
             tp = self.realize_struct(tp)
+            self.structs[obj.realtype] = tp
         self.definitions[name] = tp
 
     def add_macro(self, name, value):
diff --git a/pypy/module/cpyext/test/test_cparser.py b/pypy/module/cpyext/test/test_cparser.py
--- a/pypy/module/cpyext/test/test_cparser.py
+++ b/pypy/module/cpyext/test/test_cparser.py
@@ -204,6 +204,16 @@
     assert func_decl.get_llresult(cts) == cts.gettype('func_t*')
     assert func_decl.get_llargs(cts) == [cts.gettype('TestFloatObject *')]
 
+def test_struct_in_func_args():
+    decl = """
+    typedef struct {int x;} obj;
+    typedef int (*func)(obj x);
+    """
+    cts = parse_source(decl)
+    OBJ = cts.gettype('obj')
+    FUNCPTR = cts.gettype('func')
+    assert FUNCPTR.TO.ARGS == (OBJ,)
+
 def test_write_func():
     from ..api import ApiFunction
     from rpython.translator.c.database import LowLevelDatabase
@@ -231,7 +241,6 @@
     obj.c_x =  lltype.nullptr(rffi.CWCHARP.TO)
     lltype.free(obj, flavor='raw')
 
-
 def test_translate_cast():
     cdef = "typedef ssize_t Py_ssize_t;"
     cts = parse_source(cdef)

From pypy.commits at gmail.com  Sat Nov 16 08:29:35 2019
From: pypy.commits at gmail.com (arigo)
Date: Sat, 16 Nov 2019 05:29:35 -0800 (PST)
Subject: [pypy-commit] pypy default: Fix some failures of test_pypy_c on
 aarch64
Message-ID: <5dcff9bf.1c69fb81.5fb0c.b691@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r98072:8fc092d343a0
Date: 2019-11-16 14:28 +0100
http://bitbucket.org/pypy/pypy/changeset/8fc092d343a0/

Log:	Fix some failures of test_pypy_c on aarch64

diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -18,6 +18,20 @@
         return not detect_simd_z()
     return True
 
+def align_check(input):
+    if platform.machine().startswith('x86'):
+        return ""
+    if sys.maxsize > 2**32:
+        mask = 7
+    else:
+        mask = 3
+    return """
+        i10096 = int_and(%s, %d)
+        i10097 = int_is_zero(i10096)
+        guard_true(i10097, descr=...)
+    """ % (input, mask)
+
+
 class TestMicroNumPy(BaseTestPyPyC):
 
     arith_comb = [('+','float','float', 4*3427,   3427, 1.0,3.0),
@@ -310,6 +324,7 @@
             guard_not_invalidated(descr=...)
             i88 = int_ge(i87, i59)
             guard_false(i88, descr=...)
+            %(align_check)s
             f90 = raw_load_f(i67, i89, descr=<ArrayF 8>)
             i91 = int_add(i87, 1)
             i93 = int_add(i89, 8)
@@ -320,7 +335,7 @@
             i96 = int_lt(i95, 0)
             guard_false(i96, descr=...)
             jump(..., descr=...)
-        """)
+        """ % {"align_check": align_check('i89')})
 
     def test_array_flatiter_getitem_single(self):
         def main():
@@ -342,6 +357,7 @@
             guard_true(i126, descr=...)
             i128 = int_mul(i117, i59)
             i129 = int_add(i55, i128)
+            %(align_check)s
             f149 = raw_load_f(i100, i129, descr=<ArrayF 8>)
             i151 = int_add(i117, 1)
             setfield_gc(p156, i55, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_offset .+>)
@@ -349,7 +365,7 @@
             setarrayitem_gc(p150, 0, 0, descr=<ArrayS .+>)
             --TICK--
             jump(..., descr=...)
-        """)
+        """ % {'align_check': align_check('i129')})
 
     def test_array_flatiter_setitem_single(self):
         def main():
@@ -372,6 +388,7 @@
             i131 = int_mul(i120, i57)
             i132 = int_add(i53, i131)
             guard_not_invalidated(descr=...)
+            %(align_check)s
             raw_store(i103, i132, 42.000000, descr=<ArrayF 8>)
             i153 = int_add(i120, 1)
             i154 = getfield_raw_i(#, descr=<FieldS pypysig_long_struct.c_value 0>)
@@ -381,7 +398,7 @@
             i157 = int_lt(i154, 0)
             guard_false(i157, descr=...)
             jump(..., descr=...)
-        """)
+        """ % {'align_check': align_check('i132')})
 
     def test_mixed_div(self):
         N = 1500

From pypy.commits at gmail.com  Sat Nov 16 11:01:53 2019
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 16 Nov 2019 08:01:53 -0800 (PST)
Subject: [pypy-commit] pypy hpy: Add a cparser test
Message-ID: <5dd01d71.1c69fb81.23835.a9ab@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: hpy
Changeset: r98074:01381639f0d0
Date: 2019-11-16 16:45 +0100
http://bitbucket.org/pypy/pypy/changeset/01381639f0d0/

Log:	Add a cparser test

diff --git a/pypy/module/cpyext/test/test_cparser.py b/pypy/module/cpyext/test/test_cparser.py
--- a/pypy/module/cpyext/test/test_cparser.py
+++ b/pypy/module/cpyext/test/test_cparser.py
@@ -118,6 +118,17 @@
     Object = cts.gettype('Object')
     assert isinstance(Object, lltype.Struct)
 
+def test_incomplete_struct():
+    cdef = """
+    typedef struct s *ptr;
+    struct s {void* x;};
+    typedef struct s __s;  // HACK!
+    """
+    cts = parse_source(cdef)
+    PTR = cts.gettype("ptr")
+    assert isinstance(PTR.TO, lltype.Struct)
+    hash(PTR)
+
 def test_recursive():
     cdef = """
     typedef ssize_t Py_ssize_t;

From pypy.commits at gmail.com  Sat Nov 16 11:01:55 2019
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 16 Nov 2019 08:01:55 -0800 (PST)
Subject: [pypy-commit] pypy hpy: Begin implementing hpy_universal.load()
Message-ID: <5dd01d73.1c69fb81.6142f.6768@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: hpy
Changeset: r98075:54d6b14f3362
Date: 2019-11-16 17:01 +0100
http://bitbucket.org/pypy/pypy/changeset/54d6b14f3362/

Log:	Begin implementing hpy_universal.load()

diff --git a/pypy/module/hpy_universal/interp_hpy.py b/pypy/module/hpy_universal/interp_hpy.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/interp_hpy.py
@@ -0,0 +1,36 @@
+from rpython.rtyper.lltypesystem import rffi
+from rpython.rlib.rdynload import dlopen, dlsym, DLOpenError
+
+from pypy.interpreter.gateway import unwrap_spec
+from pypy.interpreter.error import raise_import_error
+
+
+def create_hpy_module(space, name, origin, lib, initfunc):
+    state = space.fromcache(State)
+    ctx = state.context
+    h_module = generic_cpy_call_dont_convert_result(space, initfunc, ctx)
+    return from_hpy(h_module)
+
+ at unwrap_spec(origin='fsencode', init_name='text')
+def descr_load(space, origin, init_name):
+    # XXX: this looks a lot like cpyext.api.create_extension_module()
+    name = init_name[len('HPyInit_'):]
+    try:
+        with rffi.scoped_str2charp(origin) as ll_libname:
+            lib = dlopen(ll_libname, space.sys.dlopenflags)
+    except DLOpenError as e:
+        w_path = space.newfilename(origin)
+        w_name = space.newtext(name)
+        raise raise_import_error(space,
+            space.newfilename(e.msg), w_name, w_path)
+
+    try:
+        initptr = dlsym(lib, init_name)
+    except KeyError:
+        msg = b"function %s not found in library %s" % (
+            init_name, space.utf8_w(space.newfilename(origin)))
+        w_path = space.newfilename(origin)
+        w_name = space.newtext(name)
+        raise raise_import_error(
+            space, space.newtext(msg), w_name, w_path)
+    return create_hpy_module(space, name, origin, lib, initptr)
diff --git a/pypy/module/hpy_universal/moduledef.py b/pypy/module/hpy_universal/moduledef.py
--- a/pypy/module/hpy_universal/moduledef.py
+++ b/pypy/module/hpy_universal/moduledef.py
@@ -1,5 +1,7 @@
 from pypy.interpreter.mixedmodule import MixedModule
 
 class Module(MixedModule):
-    interpleveldefs ={}
+    interpleveldefs = {
+        'load': 'interp_hpy.descr_load'
+    }
     appleveldefs = {}

From pypy.commits at gmail.com  Sat Nov 16 11:58:15 2019
From: pypy.commits at gmail.com (arigo)
Date: Sat, 16 Nov 2019 08:58:15 -0800 (PST)
Subject: [pypy-commit] pypy hpy: (ronan, arigo)
Message-ID: <5dd02aa7.1c69fb81.774bc.8eea@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: hpy
Changeset: r98076:30944126f7e1
Date: 2019-11-16 17:55 +0100
http://bitbucket.org/pypy/pypy/changeset/30944126f7e1/

Log:	(ronan, arigo)

	Don't call PyErr_Occurred() when its result is not needed

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1880,7 +1880,10 @@
         assert cpyext_glob_tid_ptr[0] == 0
         cpyext_glob_tid_ptr[0] = tid
 
-        preexist_error = PyErr_Occurred(space)
+        if convert_result and is_PyObject(RESULT_TYPE):
+            preexist_error = PyErr_Occurred(space)
+        else:
+            preexist_error = "this is not used"
         try:
             # Call the function
             result = call_external_function(func, *boxed_args)

From pypy.commits at gmail.com  Sat Nov 16 11:58:17 2019
From: pypy.commits at gmail.com (arigo)
Date: Sat, 16 Nov 2019 08:58:17 -0800 (PST)
Subject: [pypy-commit] pypy hpy: (ronan, arigo)
Message-ID: <5dd02aa9.1c69fb81.79406.ecdf@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: hpy
Changeset: r98077:5d81fcd424a5
Date: 2019-11-16 17:57 +0100
http://bitbucket.org/pypy/pypy/changeset/5d81fcd424a5/

Log:	(ronan, arigo)

	in-progress: start to make C stuff and link it to RPython stuff (for
	now, that's mostly the HPyContext)

diff --git a/pypy/module/hpy_universal/interp_hpy.py b/pypy/module/hpy_universal/interp_hpy.py
--- a/pypy/module/hpy_universal/interp_hpy.py
+++ b/pypy/module/hpy_universal/interp_hpy.py
@@ -1,19 +1,36 @@
-from rpython.rtyper.lltypesystem import rffi
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.rlib.rdynload import dlopen, dlsym, DLOpenError
 
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.error import raise_import_error
 
+from pypy.module.hpy_universal import llapi
+from pypy.module.cpyext.api import generic_cpy_call_dont_convert_result
+
+
+class State:
+    def __init__(self, space):
+        "NOT_RPYTHON"
+        self.space = space
+        self.ctx = llmemory.NULL
+
+    def setup(self):
+        if not self.ctx:
+            #_HPy_FillFunction(0, HPy_ModuleCreate)
+            self.ctx = llapi._HPy_GetGlobalCtx()
+
 
 def create_hpy_module(space, name, origin, lib, initfunc):
     state = space.fromcache(State)
-    ctx = state.context
-    h_module = generic_cpy_call_dont_convert_result(space, initfunc, ctx)
+    initfunc = rffi.cast(llapi.HPyInitFuncPtr, initfunc)
+    h_module = generic_cpy_call_dont_convert_result(space, initfunc, state.ctx)
     return from_hpy(h_module)
 
 @unwrap_spec(origin='fsencode', init_name='text')
 def descr_load(space, origin, init_name):
     # XXX: this looks a lot like cpyext.api.create_extension_module()
+    state = space.fromcache(State)
+    state.setup()
     name = init_name[len('HPyInit_'):]
     try:
         with rffi.scoped_str2charp(origin) as ll_libname:
diff --git a/pypy/module/hpy_universal/llapi.py b/pypy/module/hpy_universal/llapi.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/llapi.py
@@ -0,0 +1,62 @@
+import os
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+
+
+HPy = lltype.Signed
+HPyContext = llmemory.Address
+
+HPyInitFuncPtr = lltype.Ptr(lltype.FuncType([HPyContext], HPy))
+
+
+# ----------------------------------------------------------------
+
+# XXX temporary location
+INCLUDE_DIR = os.path.join(os.path.dirname(__file__),
+                           "test", "_vendored", "include")
+
+eci = ExternalCompilationInfo(includes=["universal/hpy.h"],
+                              include_dirs=[INCLUDE_DIR],
+                              post_include_bits=["""
+
+RPY_EXTERN void _HPy_FillFunction(int index, void *function);
+RPY_EXTERN void *_HPy_GetGlobalCtx(void);
+"""],
+                              separate_module_sources=["""
+
+struct _HPyRawContext_s {
+    int ctx_version;
+    void *ctx_raw_functions[1];
+};
+
+union _HPyContext_s_union {
+    struct _HPyContext_s ctx;
+    struct _HPyRawContext_s rawctx;
+};
+
+union _HPyContext_s_union hpy_global_ctx = {
+    {
+        .ctx_version = 1,
+    }
+};
+
+void _HPy_FillFunction(int index, void *function)
+{
+    hpy_global_ctx.rawctx.ctx_raw_functions[index] = function;
+}
+
+void *_HPy_GetGlobalCtx(void)
+{
+    return &hpy_global_ctx;
+}
+
+"""])
+
+
+_HPy_FillFunction = rffi.llexternal('_HPy_FillFunction',
+                                    [rffi.INT_real, llmemory.Address],
+                                    lltype.Void,
+                                    compilation_info=eci, _nowrapper=True)
+
+_HPy_GetGlobalCtx = rffi.llexternal('_HPy_GetGlobalCtx', [], HPyContext,
+                                    compilation_info=eci, _nowrapper=True)
diff --git a/pypy/module/hpy_universal/moduledef.py b/pypy/module/hpy_universal/moduledef.py
--- a/pypy/module/hpy_universal/moduledef.py
+++ b/pypy/module/hpy_universal/moduledef.py
@@ -1,7 +1,9 @@
 from pypy.interpreter.mixedmodule import MixedModule
 
 class Module(MixedModule):
+
+    appleveldefs = {}
+
     interpleveldefs = {
         'load': 'interp_hpy.descr_load'
     }
-    appleveldefs = {}

From pypy.commits at gmail.com  Sat Nov 16 12:58:11 2019
From: pypy.commits at gmail.com (arigo)
Date: Sat, 16 Nov 2019 09:58:11 -0800 (PST)
Subject: [pypy-commit] pypy hpy: (ronan, arigo, antocuni just arrived now)
Message-ID: <5dd038b3.1c69fb81.c02f3.1201@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: hpy
Changeset: r98078:b3357dcf16a7
Date: 2019-11-16 18:57 +0100
http://bitbucket.org/pypy/pypy/changeset/b3357dcf16a7/

Log:	(ronan, arigo, antocuni just arrived now)

	Yay, we can now import the empty module

diff --git a/pypy/module/hpy_universal/handles.py b/pypy/module/hpy_universal/handles.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/handles.py
@@ -0,0 +1,32 @@
+
+
+class HandleManager:
+
+    def __init__(self, space):
+        self.handles_w = [None]
+        self.free_list = []
+
+    def new(self, w_object):
+        if len(self.free_list) == 0:
+            index = len(self.handles_w)
+            self.handles_w.append(w_object)
+        else:
+            index = self.free_list.pop()
+            self.handles_w[index] = w_object
+        return index
+
+    def consume(self, index):
+        assert index > 0
+        w_object = self.handles_w[index]
+        self.handles_w[index] = None
+        self.free_list.append(index)
+        return w_object
+
+
+def new(space, w_object):
+    mgr = space.fromcache(HandleManager)
+    return mgr.new(w_object)
+
+def consume(space, index):
+    mgr = space.fromcache(HandleManager)
+    return mgr.consume(index)
diff --git a/pypy/module/hpy_universal/interp_hpy.py b/pypy/module/hpy_universal/interp_hpy.py
--- a/pypy/module/hpy_universal/interp_hpy.py
+++ b/pypy/module/hpy_universal/interp_hpy.py
@@ -1,30 +1,43 @@
-from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
+from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib.rdynload import dlopen, dlsym, DLOpenError
 
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.error import raise_import_error
+from pypy.interpreter.module import Module
 
-from pypy.module.hpy_universal import llapi
+from pypy.module.hpy_universal import llapi, handles
 from pypy.module.cpyext.api import generic_cpy_call_dont_convert_result
+from pypy.module.cpyext.api import slot_function
 
 
 class State:
     def __init__(self, space):
         "NOT_RPYTHON"
         self.space = space
-        self.ctx = llmemory.NULL
+        self.ctx = lltype.nullptr(rffi.VOIDP.TO)
 
     def setup(self):
         if not self.ctx:
-            #_HPy_FillFunction(0, HPy_ModuleCreate)
+            space = self.space
+            funcptr = HPyModule_Create.api_func.get_llhelper(space)
+            llapi._HPy_FillFunction(rffi.cast(rffi.INT_real, 0),
+                                    rffi.cast(rffi.VOIDP, funcptr))
             self.ctx = llapi._HPy_GetGlobalCtx()
 
 
+ at slot_function([llapi.HPyContext, lltype.Ptr(llapi.HPyModuleDef)],
+               llapi.HPy, error=0)
+def HPyModule_Create(space, ctx, hpydef):
+    modname = rffi.charp2str(hpydef.c_m_name)
+    w_mod = Module(space, space.newtext(modname))
+    return handles.new(space, w_mod)
+
+
 def create_hpy_module(space, name, origin, lib, initfunc):
     state = space.fromcache(State)
     initfunc = rffi.cast(llapi.HPyInitFuncPtr, initfunc)
     h_module = generic_cpy_call_dont_convert_result(space, initfunc, state.ctx)
-    return from_hpy(h_module)
+    return handles.consume(space, h_module)
 
 @unwrap_spec(origin='fsencode', init_name='text')
 def descr_load(space, origin, init_name):
diff --git a/pypy/module/hpy_universal/llapi.py b/pypy/module/hpy_universal/llapi.py
--- a/pypy/module/hpy_universal/llapi.py
+++ b/pypy/module/hpy_universal/llapi.py
@@ -1,13 +1,36 @@
 import os
-from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
+from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 
 
 HPy = lltype.Signed
-HPyContext = llmemory.Address
+HPyContext = rffi.VOIDP
 
 HPyInitFuncPtr = lltype.Ptr(lltype.FuncType([HPyContext], HPy))
 
+_HPyCFunctionPtr = lltype.Ptr(lltype.FuncType([HPyContext, HPy, HPy], HPy))
+_HPy_CPyCFunctionPtr = rffi.VOIDP    # not used here
+
+_HPyMethodPairFuncPtr = lltype.Ptr(lltype.FuncType([
+        rffi.CArrayPtr(_HPyCFunctionPtr),
+        rffi.CArrayPtr(_HPy_CPyCFunctionPtr)],
+    lltype.Void))
+
+HPyMethodDef = rffi.CStruct('HPyMethodDef',
+    ('ml_name', rffi.CCHARP),
+    ('ml_meth', _HPyMethodPairFuncPtr),
+    ('ml_flags', rffi.INT_real),
+    ('ml_doc', rffi.CCHARP),
+)
+
+HPyModuleDef = rffi.CStruct('HPyModuleDef',
+    ('dummy', rffi.VOIDP),
+    ('m_name', rffi.CCHARP),
+    ('m_doc', rffi.CCHARP),
+    ('m_size', lltype.Signed),
+    ('m_methods', rffi.CArrayPtr(HPyMethodDef)),
+)
+
 
 # ----------------------------------------------------------------
 
@@ -54,7 +77,7 @@
 
 
 _HPy_FillFunction = rffi.llexternal('_HPy_FillFunction',
-                                    [rffi.INT_real, llmemory.Address],
+                                    [rffi.INT_real, rffi.VOIDP],
                                     lltype.Void,
                                     compilation_info=eci, _nowrapper=True)
 
diff --git a/pypy/module/hpy_universal/test/test_basic.py b/pypy/module/hpy_universal/test/test_basic.py
--- a/pypy/module/hpy_universal/test/test_basic.py
+++ b/pypy/module/hpy_universal/test/test_basic.py
@@ -11,7 +11,13 @@
             @INIT
         """)
         assert type(mod) is type(sys)
+
+    def test_empty_module_initialization(self):
+        import sys
+        mod = self.make_module("""
+            @INIT
+        """)
+        assert type(mod) is type(sys)
         assert mod.__loader__.name == 'mytest'
         assert mod.__spec__.loader is mod.__loader__
         assert mod.__file__
-

From pypy.commits at gmail.com  Sat Nov 16 13:13:34 2019
From: pypy.commits at gmail.com (arigo)
Date: Sat, 16 Nov 2019 10:13:34 -0800 (PST)
Subject: [pypy-commit] pypy hpy: Put the next test to pass
Message-ID: <5dd03c4e.1c69fb81.90708.a7c0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: hpy
Changeset: r98079:72227d40aa42
Date: 2019-11-16 19:12 +0100
http://bitbucket.org/pypy/pypy/changeset/72227d40aa42/

Log:	Put the next test to pass

diff --git a/pypy/module/hpy_universal/test/test_basic.py b/pypy/module/hpy_universal/test/test_basic.py
--- a/pypy/module/hpy_universal/test/test_basic.py
+++ b/pypy/module/hpy_universal/test/test_basic.py
@@ -21,3 +21,21 @@
         assert mod.__loader__.name == 'mytest'
         assert mod.__spec__.loader is mod.__loader__
         assert mod.__file__
+
+    def test_different_name(self):
+        mod = self.make_module("""
+            @INIT
+        """, name="foo")
+        assert mod.__name__ == "foo"
+
+    def test_noop_function(self):
+        mod = self.make_module("""
+            HPy_FUNCTION(f)
+            static HPy f_impl(HPyContext ctx, HPy self, HPy args)
+            {
+                return HPyNone_Get(ctx);
+            }
+            @EXPORT f METH_NOARGS
+            @INIT
+        """)
+        assert mod.f() is None

From pypy.commits at gmail.com  Sat Nov 16 14:52:54 2019
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 16 Nov 2019 11:52:54 -0800 (PST)
Subject: [pypy-commit] pypy hpy: Replace hpy_universal.load() with
 hpy_universal.load_from_spec()
Message-ID: <5dd05396.1c69fb81.418aa.ef40@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: hpy
Changeset: r98080:b0b337fc4202
Date: 2019-11-16 20:51 +0100
http://bitbucket.org/pypy/pypy/changeset/b0b337fc4202/

Log:	Replace hpy_universal.load() with hpy_universal.load_from_spec()

diff --git a/pypy/module/hpy_universal/interp_hpy.py b/pypy/module/hpy_universal/interp_hpy.py
--- a/pypy/module/hpy_universal/interp_hpy.py
+++ b/pypy/module/hpy_universal/interp_hpy.py
@@ -39,28 +39,29 @@
     h_module = generic_cpy_call_dont_convert_result(space, initfunc, state.ctx)
     return handles.consume(space, h_module)
 
- at unwrap_spec(origin='fsencode', init_name='text')
-def descr_load(space, origin, init_name):
+def descr_load_from_spec(space, w_spec):
     # XXX: this looks a lot like cpyext.api.create_extension_module()
     state = space.fromcache(State)
     state.setup()
-    name = init_name[len('HPyInit_'):]
+    w_name = space.getattr(w_spec, space.newtext("name"))
+    name = space.text_w(w_name)
+    origin = space.text_w(space.getattr(w_spec, space.newtext("origin")))
     try:
         with rffi.scoped_str2charp(origin) as ll_libname:
             lib = dlopen(ll_libname, space.sys.dlopenflags)
     except DLOpenError as e:
         w_path = space.newfilename(origin)
-        w_name = space.newtext(name)
         raise raise_import_error(space,
             space.newfilename(e.msg), w_name, w_path)
 
+    basename = name.split('.')[-1]
+    init_name = 'HPyInit_' + basename
     try:
         initptr = dlsym(lib, init_name)
     except KeyError:
         msg = b"function %s not found in library %s" % (
             init_name, space.utf8_w(space.newfilename(origin)))
         w_path = space.newfilename(origin)
-        w_name = space.newtext(name)
         raise raise_import_error(
             space, space.newtext(msg), w_name, w_path)
     return create_hpy_module(space, name, origin, lib, initptr)
diff --git a/pypy/module/hpy_universal/moduledef.py b/pypy/module/hpy_universal/moduledef.py
--- a/pypy/module/hpy_universal/moduledef.py
+++ b/pypy/module/hpy_universal/moduledef.py
@@ -5,5 +5,5 @@
     appleveldefs = {}
 
     interpleveldefs = {
-        'load': 'interp_hpy.descr_load'
+        'load_from_spec': 'interp_hpy.descr_load_from_spec'
     }
diff --git a/pypy/module/hpy_universal/test/support.py b/pypy/module/hpy_universal/test/support.py
--- a/pypy/module/hpy_universal/test/support.py
+++ b/pypy/module/hpy_universal/test/support.py
@@ -25,6 +25,14 @@
             pytest.skip()
         cls.compiler = ExtensionCompiler(udir)
 
+        w_FakeSpec = cls.space.appexec([], """():
+            class FakeSpec:
+                def __init__(self, name, origin):
+                    self.name = name
+                    self.origin = origin
+            return FakeSpec
+        """)
+
         @unwrap_spec(source_template='text', name='text')
         def descr_make_module(space, source_template, name='mytest'):
             source = _support.expand_template(source_template, name)
@@ -38,10 +46,10 @@
                                     universal_mode=True)
             #
             w_mod = space.appexec(
-                [space.newtext(so_filename), space.newtext(name)],
-                """(path, modname):
-                    from hpy_universal import load
-                    return load(path, 'HPyInit_' + modname)
+                [w_FakeSpec, space.newtext(so_filename), space.newtext(name)],
+                """(FakeSpec, path, modname):
+                    from hpy_universal import load_from_spec
+                    return load_from_spec(FakeSpec(modname, path))
                 """
             )
             return w_mod

From pypy.commits at gmail.com  Sat Nov 16 15:17:07 2019
From: pypy.commits at gmail.com (arigo)
Date: Sat, 16 Nov 2019 12:17:07 -0800 (PST)
Subject: [pypy-commit] pypy hpy: (antocuni, ronan, arigo)
Message-ID: <5dd05943.1c69fb81.43ca8.b96b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: hpy
Changeset: r98081:3612658138c2
Date: 2019-11-16 21:16 +0100
http://bitbucket.org/pypy/pypy/changeset/3612658138c2/

Log:	(antocuni, ronan, arigo)

	Progress towards, but not yet, reaching test_noop_function

diff --git a/pypy/module/hpy_universal/interp_extfunc.py b/pypy/module/hpy_universal/interp_extfunc.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/interp_extfunc.py
@@ -0,0 +1,79 @@
+from rpython.rtyper.lltypesystem import lltype, rffi
+from pypy.interpreter.error import oefmt
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.typedef import TypeDef, interp2app
+
+from pypy.module.hpy_universal import llapi, handles
+from pypy.module.hpy_universal.state import State
+from pypy.module.cpyext.api import generic_cpy_call_dont_convert_result
+
+
+class W_ExtensionFunction(W_Root):
+    _immutable_fields_ = ["flags", "name"]
+
+    def __init__(self, ml):
+        self.ml = ml
+        self.name = rffi.charp2str(self.ml.c_ml_name)
+        self.flags = rffi.cast(lltype.Signed, self.ml.c_ml_flags)
+        # fetch the real HPy function pointer, by calling ml_meth, which
+        # is a function that returns it and also the CPython-only trampoline
+        with lltype.scoped_alloc(
+                rffi.CArray(llapi._HPyCFunctionPtr), 1) as funcptr:
+            with lltype.scoped_alloc(
+                    rffi.CArray(llapi._HPy_CPyCFunctionPtr), 1)  \
+                                                        as ignored_trampoline:
+                ml.c_ml_meth(funcptr, ignored_trampoline)
+                self.cfuncptr = funcptr[0]
+
+    def call_keywords(self, space, __args__):
+        raise NotImplementedError("later")
+
+    def call_noargs(self, space):
+        state = space.fromcache(State)
+        h_result = generic_cpy_call_dont_convert_result(space, self.cfuncptr,
+            state.ctx, 0, 0)
+        # XXX check for exceptions
+        return handles.consume(h_result)
+
+    def call_o(self, space, w_arg):
+        raise NotImplementedError("later")
+
+    def call_varargs(self, space, arguments_w):
+        raise NotImplementedError("later")
+
+    def descr_call(self, space, __args__):
+        flags = self.flags
+        length = len(__args__.arguments_w)
+
+        if flags & llapi.METH_KEYWORDS:
+            return self.call_keywords(space, __args__)
+
+        if __args__.keywords:
+            raise oefmt(space.w_TypeError,
+                        "%s() takes no keyword arguments", self.name)
+
+        if flags & llapi.METH_NOARGS:
+            if length == 0:
+                return self.call_noargs(space)
+            raise oefmt(space.w_TypeError,
+                        "%s() takes no arguments", self.name)
+
+        if flags & llapi.METH_O:
+            if length != 1:
+                raise oefmt(space.w_TypeError,
+                            "%s() takes exactly one argument (%d given)",
+                            self.name, length)
+            return self.call_o(space, __args__.arguments_w[0])
+
+        if flags & llapi.METH_VARARGS:
+            return self.call_varargs(space, __args__.arguments_w)
+        else:  # shouldn't happen!
+            raise oefmt(space.w_RuntimeError, "unknown calling convention")
+
+
+
+W_ExtensionFunction.typedef = TypeDef(
+    'extension_function',
+    __call__ = interp2app(W_ExtensionFunction.descr_call),
+    )
+W_ExtensionFunction.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/hpy_universal/interp_hpy.py b/pypy/module/hpy_universal/interp_hpy.py
--- a/pypy/module/hpy_universal/interp_hpy.py
+++ b/pypy/module/hpy_universal/interp_hpy.py
@@ -5,31 +5,27 @@
 from pypy.interpreter.error import raise_import_error
 from pypy.interpreter.module import Module
 
-from pypy.module.hpy_universal import llapi, handles
+from pypy.module.hpy_universal import llapi, handles, interp_extfunc
+from pypy.module.hpy_universal.state import State
 from pypy.module.cpyext.api import generic_cpy_call_dont_convert_result
 from pypy.module.cpyext.api import slot_function
 
 
-class State:
-    def __init__(self, space):
-        "NOT_RPYTHON"
-        self.space = space
-        self.ctx = lltype.nullptr(rffi.VOIDP.TO)
-
-    def setup(self):
-        if not self.ctx:
-            space = self.space
-            funcptr = HPyModule_Create.api_func.get_llhelper(space)
-            llapi._HPy_FillFunction(rffi.cast(rffi.INT_real, 0),
-                                    rffi.cast(rffi.VOIDP, funcptr))
-            self.ctx = llapi._HPy_GetGlobalCtx()
-
-
 @slot_function([llapi.HPyContext, lltype.Ptr(llapi.HPyModuleDef)],
                llapi.HPy, error=0)
 def HPyModule_Create(space, ctx, hpydef):
     modname = rffi.charp2str(hpydef.c_m_name)
     w_mod = Module(space, space.newtext(modname))
+    #
+    # add all the functions defined in hpydef.c_m_methods
+    if hpydef.c_m_methods:
+        p = hpydef.c_m_methods
+        i = 0
+        while p[i].c_ml_name:
+            w_extfunc = interp_extfunc.W_ExtensionFunction(p[i])
+            space.setattr(w_mod, space.newtext(w_extfunc.name), w_extfunc)
+            i += 1
+    #
     return handles.new(space, w_mod)
 
 
diff --git a/pypy/module/hpy_universal/llapi.py b/pypy/module/hpy_universal/llapi.py
--- a/pypy/module/hpy_universal/llapi.py
+++ b/pypy/module/hpy_universal/llapi.py
@@ -4,7 +4,21 @@
 
 
 HPy = lltype.Signed
-HPyContext = rffi.VOIDP
+HPyContextS = lltype.Struct('dummy_HPyContext_s',
+    ('ctx_version', rffi.INT_real),
+    ('ctx_Module_Create', rffi.VOIDP),
+    ('ctx_None_Get', rffi.VOIDP),
+    ('ctx_Dup', rffi.VOIDP),
+    ('ctx_Close', rffi.VOIDP),
+    ('ctx_Long_FromLong', rffi.VOIDP),
+    ('ctx_Arg_ParseTuple', rffi.VOIDP),
+    ('ctx_Number_Add', rffi.VOIDP),
+    ('ctx_Unicode_FromString', rffi.VOIDP),
+    ('ctx_FromPyObject', rffi.VOIDP),
+    ('ctx_AsPyObject', rffi.VOIDP),
+    ('ctx_CallRealFunctionFromTrampoline', rffi.VOIDP),
+)
+HPyContext = lltype.Ptr(HPyContextS)
 
 HPyInitFuncPtr = lltype.Ptr(lltype.FuncType([HPyContext], HPy))
 
@@ -31,6 +45,11 @@
     ('m_methods', rffi.CArrayPtr(HPyMethodDef)),
 )
 
+METH_VARARGS  = 0x0001
+METH_KEYWORDS = 0x0002
+METH_NOARGS   = 0x0004
+METH_O        = 0x0008
+
 
 # ----------------------------------------------------------------
 
@@ -47,26 +66,7 @@
 """],
                               separate_module_sources=["""
 
-struct _HPyRawContext_s {
-    int ctx_version;
-    void *ctx_raw_functions[1];
-};
-
-union _HPyContext_s_union {
-    struct _HPyContext_s ctx;
-    struct _HPyRawContext_s rawctx;
-};
-
-union _HPyContext_s_union hpy_global_ctx = {
-    {
-        .ctx_version = 1,
-    }
-};
-
-void _HPy_FillFunction(int index, void *function)
-{
-    hpy_global_ctx.rawctx.ctx_raw_functions[index] = function;
-}
+struct _HPyContext_s hpy_global_ctx;
 
 void *_HPy_GetGlobalCtx(void)
 {
@@ -75,11 +75,5 @@
 
 """])
 
-
-_HPy_FillFunction = rffi.llexternal('_HPy_FillFunction',
-                                    [rffi.INT_real, rffi.VOIDP],
-                                    lltype.Void,
-                                    compilation_info=eci, _nowrapper=True)
-
 _HPy_GetGlobalCtx = rffi.llexternal('_HPy_GetGlobalCtx', [], HPyContext,
                                     compilation_info=eci, _nowrapper=True)
diff --git a/pypy/module/hpy_universal/state.py b/pypy/module/hpy_universal/state.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/state.py
@@ -0,0 +1,33 @@
+import os
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.annlowlevel import llhelper
+from pypy.module.hpy_universal import llapi
+
+
+class State:
+    def __init__(self, space):
+        "NOT_RPYTHON"
+        self.space = space
+        self.ctx = lltype.nullptr(rffi.VOIDP.TO)
+
+    def setup(self):
+        if self.ctx:
+            return
+
+        self.ctx = llapi._HPy_GetGlobalCtx()
+
+        DUMMY_FUNC = lltype.FuncType([], lltype.Void)
+        for name in llapi.HPyContext.TO._names:
+            if name != 'ctx_version':
+                def missing_function(name=name):
+                    print ("oops! calling the slot %r, which is not implemented"
+                           % (name,))
+                    os._exit(1)
+                funcptr = llhelper(lltype.Ptr(DUMMY_FUNC), missing_function)
+                setattr(self.ctx, name, rffi.cast(rffi.VOIDP, funcptr))
+
+        # XXX collect all these functions automatically
+        from pypy.module.hpy_universal import interp_hpy
+        space = self.space
+        funcptr = interp_hpy.HPyModule_Create.api_func.get_llhelper(space)
+        self.ctx.ctx_Module_Create = rffi.cast(rffi.VOIDP, funcptr)
diff --git a/pypy/module/hpy_universal/test/support.py b/pypy/module/hpy_universal/test/support.py
--- a/pypy/module/hpy_universal/test/support.py
+++ b/pypy/module/hpy_universal/test/support.py
@@ -41,7 +41,8 @@
             filename.write(source)
             #
             ext = _support.get_extension(str(filename), name, include_dirs=[INCLUDE_DIR],
-                                extra_compile_args=['-Wfatal-errors'])
+                                extra_compile_args=['-Wfatal-errors', '-g', '-Og'],
+                                extra_link_args=['-g'])
             so_filename = _support.c_compile(str(tmpdir), ext, compiler_verbose=False,
                                     universal_mode=True)
             #

From pypy.commits at gmail.com  Sat Nov 16 18:22:25 2019
From: pypy.commits at gmail.com (antocuni)
Date: Sat, 16 Nov 2019 15:22:25 -0800 (PST)
Subject: [pypy-commit] pypy hpy: (antocuni, arigo): implement HPyNone_Get,
 test_noop_function finally passes
Message-ID: <5dd084b1.1c69fb81.139f9.1b0a@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98082:bacbce0ca621
Date: 2019-11-16 23:44 +0100
http://bitbucket.org/pypy/pypy/changeset/bacbce0ca621/

Log:	(antocuni, arigo): implement HPyNone_Get, test_noop_function finally
	passes

diff --git a/pypy/module/hpy_universal/interp_extfunc.py b/pypy/module/hpy_universal/interp_extfunc.py
--- a/pypy/module/hpy_universal/interp_extfunc.py
+++ b/pypy/module/hpy_universal/interp_extfunc.py
@@ -33,7 +33,7 @@
         h_result = generic_cpy_call_dont_convert_result(space, self.cfuncptr,
             state.ctx, 0, 0)
         # XXX check for exceptions
-        return handles.consume(h_result)
+        return handles.consume(space, h_result)
 
     def call_o(self, space, w_arg):
         raise NotImplementedError("later")
diff --git a/pypy/module/hpy_universal/interp_hpy.py b/pypy/module/hpy_universal/interp_hpy.py
--- a/pypy/module/hpy_universal/interp_hpy.py
+++ b/pypy/module/hpy_universal/interp_hpy.py
@@ -29,6 +29,11 @@
     return handles.new(space, w_mod)
 
 
+ at slot_function([llapi.HPyContext], llapi.HPy, error=0)
+def HPyNone_Get(space, ctx):
+    return handles.new(space, space.w_None)
+
+
 def create_hpy_module(space, name, origin, lib, initfunc):
     state = space.fromcache(State)
     initfunc = rffi.cast(llapi.HPyInitFuncPtr, initfunc)
diff --git a/pypy/module/hpy_universal/state.py b/pypy/module/hpy_universal/state.py
--- a/pypy/module/hpy_universal/state.py
+++ b/pypy/module/hpy_universal/state.py
@@ -31,3 +31,6 @@
         space = self.space
         funcptr = interp_hpy.HPyModule_Create.api_func.get_llhelper(space)
         self.ctx.ctx_Module_Create = rffi.cast(rffi.VOIDP, funcptr)
+        #
+        funcptr = interp_hpy.HPyNone_Get.api_func.get_llhelper(space)
+        self.ctx.ctx_None_Get = rffi.cast(rffi.VOIDP, funcptr)

From pypy.commits at gmail.com  Sat Nov 16 18:22:27 2019
From: pypy.commits at gmail.com (antocuni)
Date: Sat, 16 Nov 2019 15:22:27 -0800 (PST)
Subject: [pypy-commit] pypy hpy: (antocuni, arigo,
 ronan): kill the usage of @slot_function from cpyext and use the
 much simpler @apifunc
Message-ID: <5dd084b3.1c69fb81.67a62.9d76@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98083:949c501dadd7
Date: 2019-11-17 00:21 +0100
http://bitbucket.org/pypy/pypy/changeset/949c501dadd7/

Log:	(antocuni, arigo, ronan): kill the usage of @slot_function from
	cpyext and use the much simpler @apifunc

diff --git a/pypy/module/hpy_universal/interp_hpy.py b/pypy/module/hpy_universal/interp_hpy.py
--- a/pypy/module/hpy_universal/interp_hpy.py
+++ b/pypy/module/hpy_universal/interp_hpy.py
@@ -1,3 +1,4 @@
+from rpython.rtyper.annlowlevel import llhelper
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib.rdynload import dlopen, dlsym, DLOpenError
 
@@ -8,11 +9,24 @@
 from pypy.module.hpy_universal import llapi, handles, interp_extfunc
 from pypy.module.hpy_universal.state import State
 from pypy.module.cpyext.api import generic_cpy_call_dont_convert_result
-from pypy.module.cpyext.api import slot_function
 
 
- at slot_function([llapi.HPyContext, lltype.Ptr(llapi.HPyModuleDef)],
-               llapi.HPy, error=0)
+def apifunc(argtypes, restype, error):
+    # XXX: at the moment, error is ignored. We should do something with it
+    # and handle exceptions properly
+    def decorate(fn):
+        ll_functype = lltype.Ptr(lltype.FuncType(argtypes, restype))
+        def get_llhelper(space):
+            def wrapper(*args):
+                return fn(space, *args)
+            return llhelper(ll_functype, wrapper)
+        fn.get_llhelper = get_llhelper
+        return fn
+    return decorate
+
+
+ at apifunc([llapi.HPyContext, lltype.Ptr(llapi.HPyModuleDef)],
+         llapi.HPy, error=0)
 def HPyModule_Create(space, ctx, hpydef):
     modname = rffi.charp2str(hpydef.c_m_name)
     w_mod = Module(space, space.newtext(modname))
@@ -29,7 +43,7 @@
     return handles.new(space, w_mod)
 
 
- at slot_function([llapi.HPyContext], llapi.HPy, error=0)
+ at apifunc([llapi.HPyContext], llapi.HPy, error=0)
 def HPyNone_Get(space, ctx):
     return handles.new(space, space.w_None)
 
diff --git a/pypy/module/hpy_universal/state.py b/pypy/module/hpy_universal/state.py
--- a/pypy/module/hpy_universal/state.py
+++ b/pypy/module/hpy_universal/state.py
@@ -29,8 +29,9 @@
         # XXX collect all these functions automatically
         from pypy.module.hpy_universal import interp_hpy
         space = self.space
-        funcptr = interp_hpy.HPyModule_Create.api_func.get_llhelper(space)
+        
+        funcptr = interp_hpy.HPyModule_Create.get_llhelper(space)
         self.ctx.ctx_Module_Create = rffi.cast(rffi.VOIDP, funcptr)
         #
-        funcptr = interp_hpy.HPyNone_Get.api_func.get_llhelper(space)
+        funcptr = interp_hpy.HPyNone_Get.get_llhelper(space)
         self.ctx.ctx_None_Get = rffi.cast(rffi.VOIDP, funcptr)

From pypy.commits at gmail.com  Sat Nov 16 18:29:45 2019
From: pypy.commits at gmail.com (arigo)
Date: Sat, 16 Nov 2019 15:29:45 -0800 (PST)
Subject: [pypy-commit] pypy hpy: Translation fixes. Try following these
 specialize.memo() if you can!
Message-ID: <5dd08669.1c69fb81.ecab8.76f0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: hpy
Changeset: r98084:f3ead38c0c67
Date: 2019-11-17 00:29 +0100
http://bitbucket.org/pypy/pypy/changeset/f3ead38c0c67/

Log:	Translation fixes. Try following these specialize.memo() if you can!

diff --git a/pypy/module/hpy_universal/interp_hpy.py b/pypy/module/hpy_universal/interp_hpy.py
--- a/pypy/module/hpy_universal/interp_hpy.py
+++ b/pypy/module/hpy_universal/interp_hpy.py
@@ -1,6 +1,7 @@
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib.rdynload import dlopen, dlsym, DLOpenError
+from rpython.rlib.objectmodel import specialize
 
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.error import raise_import_error
@@ -16,10 +17,13 @@
     # and handle exceptions properly
     def decorate(fn):
         ll_functype = lltype.Ptr(lltype.FuncType(argtypes, restype))
-        def get_llhelper(space):
+        @specialize.memo()
+        def make_wrapper(space):
             def wrapper(*args):
                 return fn(space, *args)
-            return llhelper(ll_functype, wrapper)
+            return wrapper
+        def get_llhelper(space):
+            return llhelper(ll_functype, make_wrapper(space))
         fn.get_llhelper = get_llhelper
         return fn
     return decorate
diff --git a/pypy/module/hpy_universal/state.py b/pypy/module/hpy_universal/state.py
--- a/pypy/module/hpy_universal/state.py
+++ b/pypy/module/hpy_universal/state.py
@@ -1,14 +1,27 @@
 import os
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.annlowlevel import llhelper
+from rpython.rlib.unroll import unrolling_iterable
+from rpython.rlib.objectmodel import specialize
 from pypy.module.hpy_universal import llapi
 
+CONTEXT_NAMES = unrolling_iterable(llapi.HPyContext.TO._names)
+DUMMY_FUNC = lltype.FuncType([], lltype.Void)
+
+ at specialize.memo()
+def make_missing_function(name):
+    def missing_function():
+        print ("oops! calling the slot '%s', "
+               "which is not implemented" % (name,))
+        os._exit(1)
+    return missing_function
+
 
 class State:
     def __init__(self, space):
         "NOT_RPYTHON"
         self.space = space
-        self.ctx = lltype.nullptr(rffi.VOIDP.TO)
+        self.ctx = lltype.nullptr(llapi.HPyContext.TO)
 
     def setup(self):
         if self.ctx:
@@ -16,13 +29,9 @@
 
         self.ctx = llapi._HPy_GetGlobalCtx()
 
-        DUMMY_FUNC = lltype.FuncType([], lltype.Void)
-        for name in llapi.HPyContext.TO._names:
+        for name in CONTEXT_NAMES:
             if name != 'ctx_version':
-                def missing_function(name=name):
-                    print ("oops! calling the slot %r, which is not implemented"
-                           % (name,))
-                    os._exit(1)
+                missing_function = make_missing_function(name)
                 funcptr = llhelper(lltype.Ptr(DUMMY_FUNC), missing_function)
                 setattr(self.ctx, name, rffi.cast(rffi.VOIDP, funcptr))
 

From pypy.commits at gmail.com  Sat Nov 16 18:32:19 2019
From: pypy.commits at gmail.com (antocuni)
Date: Sat, 16 Nov 2019 15:32:19 -0800 (PST)
Subject: [pypy-commit] pypy hpy: skip this for now
Message-ID: <5dd08703.1c69fb81.aca64.01ee@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98085:256add727d58
Date: 2019-11-17 00:31 +0100
http://bitbucket.org/pypy/pypy/changeset/256add727d58/

Log:	skip this for now

diff --git a/pypy/module/hpy_universal/test/test_basic.py b/pypy/module/hpy_universal/test/test_basic.py
--- a/pypy/module/hpy_universal/test/test_basic.py
+++ b/pypy/module/hpy_universal/test/test_basic.py
@@ -13,6 +13,7 @@
         assert type(mod) is type(sys)
 
     def test_empty_module_initialization(self):
+        skip("FIXME")
         import sys
         mod = self.make_module("""
             @INIT

From pypy.commits at gmail.com  Sat Nov 16 19:16:15 2019
From: pypy.commits at gmail.com (antocuni)
Date: Sat, 16 Nov 2019 16:16:15 -0800 (PST)
Subject: [pypy-commit] pypy hpy: progress towards passing
 test_self_is_module: implement HPy_Dup
Message-ID: <5dd0914f.1c69fb81.2b7b7.cef7@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98086:33150b74ce8b
Date: 2019-11-17 00:37 +0100
http://bitbucket.org/pypy/pypy/changeset/33150b74ce8b/

Log:	progress towards passing test_self_is_module: implement HPy_Dup

diff --git a/pypy/module/hpy_universal/handles.py b/pypy/module/hpy_universal/handles.py
--- a/pypy/module/hpy_universal/handles.py
+++ b/pypy/module/hpy_universal/handles.py
@@ -22,6 +22,10 @@
         self.free_list.append(index)
         return w_object
 
+    def dup(self, index):
+        w_object = self.handles_w[index]
+        return self.new(w_object)
+
 
 def new(space, w_object):
     mgr = space.fromcache(HandleManager)
@@ -30,3 +34,7 @@
 def consume(space, index):
     mgr = space.fromcache(HandleManager)
     return mgr.consume(index)
+
+def dup(space, index):
+    mgr = space.fromcache(HandleManager)
+    return mgr.dup(index)
diff --git a/pypy/module/hpy_universal/interp_hpy.py b/pypy/module/hpy_universal/interp_hpy.py
--- a/pypy/module/hpy_universal/interp_hpy.py
+++ b/pypy/module/hpy_universal/interp_hpy.py
@@ -51,6 +51,10 @@
 def HPyNone_Get(space, ctx):
     return handles.new(space, space.w_None)
 
+ at apifunc([llapi.HPyContext, llapi.HPy], llapi.HPy, error=0)
+def HPy_Dup(space, ctx, h):
+    return handles.dup(space, h)
+
 
 def create_hpy_module(space, name, origin, lib, initfunc):
     state = space.fromcache(State)
diff --git a/pypy/module/hpy_universal/state.py b/pypy/module/hpy_universal/state.py
--- a/pypy/module/hpy_universal/state.py
+++ b/pypy/module/hpy_universal/state.py
@@ -44,3 +44,6 @@
         #
         funcptr = interp_hpy.HPyNone_Get.get_llhelper(space)
         self.ctx.ctx_None_Get = rffi.cast(rffi.VOIDP, funcptr)
+        #
+        funcptr = interp_hpy.HPy_Dup.get_llhelper(space)
+        self.ctx.ctx_Dup = rffi.cast(rffi.VOIDP, funcptr)
diff --git a/pypy/module/hpy_universal/test/test_basic.py b/pypy/module/hpy_universal/test/test_basic.py
--- a/pypy/module/hpy_universal/test/test_basic.py
+++ b/pypy/module/hpy_universal/test/test_basic.py
@@ -40,3 +40,15 @@
             @INIT
         """)
         assert mod.f() is None
+
+    def test_self_is_module(self):
+        mod = self.make_module("""
+            HPy_FUNCTION(f)
+            static HPy f_impl(HPyContext ctx, HPy self, HPy args)
+            {
+                return HPy_Dup(ctx, self);
+            }
+            @EXPORT f METH_NOARGS
+            @INIT
+        """)
+        assert mod.f() is mod

From pypy.commits at gmail.com  Sat Nov 16 19:16:17 2019
From: pypy.commits at gmail.com (antocuni)
Date: Sat, 16 Nov 2019 16:16:17 -0800 (PST)
Subject: [pypy-commit] pypy hpy: pass the module as 'self' when calling
 HPyFunctionDefs. test_self_is_module passes
Message-ID: <5dd09151.1c69fb81.9dc83.45bf@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98087:1cbaaff6fda2
Date: 2019-11-17 00:43 +0100
http://bitbucket.org/pypy/pypy/changeset/1cbaaff6fda2/

Log:	pass the module as 'self' when calling HPyFunctionDefs.
	test_self_is_module passes

diff --git a/pypy/module/hpy_universal/interp_extfunc.py b/pypy/module/hpy_universal/interp_extfunc.py
--- a/pypy/module/hpy_universal/interp_extfunc.py
+++ b/pypy/module/hpy_universal/interp_extfunc.py
@@ -11,8 +11,9 @@
 class W_ExtensionFunction(W_Root):
     _immutable_fields_ = ["flags", "name"]
 
-    def __init__(self, ml):
+    def __init__(self, ml, w_self):
         self.ml = ml
+        self.w_self = w_self
         self.name = rffi.charp2str(self.ml.c_ml_name)
         self.flags = rffi.cast(lltype.Signed, self.ml.c_ml_flags)
         # fetch the real HPy function pointer, by calling ml_meth, which
@@ -30,8 +31,10 @@
 
     def call_noargs(self, space):
         state = space.fromcache(State)
+        h_self = handles.new(space, self.w_self)
         h_result = generic_cpy_call_dont_convert_result(space, self.cfuncptr,
-            state.ctx, 0, 0)
+            state.ctx, h_self, 0)
+        handles.consume(space, h_self)
         # XXX check for exceptions
         return handles.consume(space, h_result)
 
diff --git a/pypy/module/hpy_universal/interp_hpy.py b/pypy/module/hpy_universal/interp_hpy.py
--- a/pypy/module/hpy_universal/interp_hpy.py
+++ b/pypy/module/hpy_universal/interp_hpy.py
@@ -40,7 +40,7 @@
         p = hpydef.c_m_methods
         i = 0
         while p[i].c_ml_name:
-            w_extfunc = interp_extfunc.W_ExtensionFunction(p[i])
+            w_extfunc = interp_extfunc.W_ExtensionFunction(p[i], w_mod)
             space.setattr(w_mod, space.newtext(w_extfunc.name), w_extfunc)
             i += 1
     #

From pypy.commits at gmail.com  Sat Nov 16 19:16:18 2019
From: pypy.commits at gmail.com (antocuni)
Date: Sat, 16 Nov 2019 16:16:18 -0800 (PST)
Subject: [pypy-commit] pypy hpy: add unit test for handles.py
Message-ID: <5dd09152.1c69fb81.5ec18.833a@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98088:3308dc2cfbc4
Date: 2019-11-17 00:49 +0100
http://bitbucket.org/pypy/pypy/changeset/3308dc2cfbc4/

Log:	add unit test for handles.py

diff --git a/pypy/module/hpy_universal/test/test_handles.py b/pypy/module/hpy_universal/test/test_handles.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/test/test_handles.py
@@ -0,0 +1,31 @@
+from pypy.module.hpy_universal.handles import HandleManager
+
+class TestHandleManager(object):
+
+    def test_new(self):
+        mgr = HandleManager(None)
+        h = mgr.new('hello')
+        assert mgr.handles_w[h] == 'hello'
+
+    def test_consume(self):
+        mgr = HandleManager(None)
+        h = mgr.new('hello')
+        assert mgr.consume(h) == 'hello'
+        assert mgr.handles_w[h] is None
+
+    def test_freelist(self):
+        mgr = HandleManager(None)
+        h0 = mgr.new('hello')
+        h1 = mgr.new('world')
+        assert mgr.consume(h0) == 'hello'
+        assert mgr.free_list == [h0]
+        h2 = mgr.new('hello2')
+        assert h2 == h0
+        assert mgr.free_list == []
+
+    def test_dup(self):
+        mgr = HandleManager(None)
+        h0 = mgr.new('hello')
+        h1 = mgr.dup(h0)
+        assert h1 != h0
+        assert mgr.consume(h0) == mgr.consume(h1) == 'hello'

From pypy.commits at gmail.com  Sat Nov 16 19:16:20 2019
From: pypy.commits at gmail.com (antocuni)
Date: Sat, 16 Nov 2019 16:16:20 -0800 (PST)
Subject: [pypy-commit] pypy hpy: implement handles.close
Message-ID: <5dd09154.1c69fb81.774bc.ec0e@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98089:a117a040ec4f
Date: 2019-11-17 00:52 +0100
http://bitbucket.org/pypy/pypy/changeset/a117a040ec4f/

Log:	implement handles.close

diff --git a/pypy/module/hpy_universal/handles.py b/pypy/module/hpy_universal/handles.py
--- a/pypy/module/hpy_universal/handles.py
+++ b/pypy/module/hpy_universal/handles.py
@@ -15,11 +15,18 @@
             self.handles_w[index] = w_object
         return index
 
+    def close(self, index):
+        assert index > 0
+        self.handles_w[index] = None
+        self.free_list.append(index)
+
     def consume(self, index):
+        """
+        Like close, but also return the w_object which was pointed by the handled
+        """
         assert index > 0
         w_object = self.handles_w[index]
-        self.handles_w[index] = None
-        self.free_list.append(index)
+        self.close(index)
         return w_object
 
     def dup(self, index):
diff --git a/pypy/module/hpy_universal/test/test_handles.py b/pypy/module/hpy_universal/test/test_handles.py
--- a/pypy/module/hpy_universal/test/test_handles.py
+++ b/pypy/module/hpy_universal/test/test_handles.py
@@ -7,6 +7,12 @@
         h = mgr.new('hello')
         assert mgr.handles_w[h] == 'hello'
 
+    def test_close(self):
+        mgr = HandleManager(None)
+        h = mgr.new('hello')
+        assert mgr.close(h) is None
+        assert mgr.handles_w[h] is None
+
     def test_consume(self):
         mgr = HandleManager(None)
         h = mgr.new('hello')

From pypy.commits at gmail.com  Sat Nov 16 19:16:22 2019
From: pypy.commits at gmail.com (antocuni)
Date: Sat, 16 Nov 2019 16:16:22 -0800 (PST)
Subject: [pypy-commit] pypy hpy: implement a context-manager to allocate
 handles
Message-ID: <5dd09156.1c69fb81.18797.d8a1@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98090:1dc8e3639eb3
Date: 2019-11-17 01:00 +0100
http://bitbucket.org/pypy/pypy/changeset/1dc8e3639eb3/

Log:	implement a context-manager to allocate handles

diff --git a/pypy/module/hpy_universal/handles.py b/pypy/module/hpy_universal/handles.py
--- a/pypy/module/hpy_universal/handles.py
+++ b/pypy/module/hpy_universal/handles.py
@@ -38,6 +38,10 @@
     mgr = space.fromcache(HandleManager)
     return mgr.new(w_object)
 
+def close(space, index):
+    mgr = space.fromcache(HandleManager)
+    mgr.close(index)
+    
 def consume(space, index):
     mgr = space.fromcache(HandleManager)
     return mgr.consume(index)
@@ -45,3 +49,21 @@
 def dup(space, index):
     mgr = space.fromcache(HandleManager)
     return mgr.dup(index)
+
+
+class using(object):
+    """
+    context-manager to new/close a handle
+    """
+
+    def __init__(self, space, w_object):
+        self.space = space
+        self.w_object = w_object
+        self.h = -1
+
+    def __enter__(self):
+        self.h = new(self.space, self.w_object)
+        return self.h
+
+    def __exit__(self, etype, evalue, tb):
+        close(self.space, self.h)
diff --git a/pypy/module/hpy_universal/test/test_handles.py b/pypy/module/hpy_universal/test/test_handles.py
--- a/pypy/module/hpy_universal/test/test_handles.py
+++ b/pypy/module/hpy_universal/test/test_handles.py
@@ -1,3 +1,4 @@
+from pypy.module.hpy_universal import handles
 from pypy.module.hpy_universal.handles import HandleManager
 
 class TestHandleManager(object):
@@ -35,3 +36,14 @@
         h1 = mgr.dup(h0)
         assert h1 != h0
         assert mgr.consume(h0) == mgr.consume(h1) == 'hello'
+
+def test_using():
+    mgr = HandleManager(None)
+    class FakeSpace(object):
+        @classmethod
+        def fromcache(cls, x):
+            return mgr
+    space = FakeSpace()
+    with handles.using(space, 'hello') as h:
+        assert mgr.handles_w[h] == 'hello'
+    assert mgr.handles_w[h] is None

From pypy.commits at gmail.com  Sat Nov 16 19:16:23 2019
From: pypy.commits at gmail.com (antocuni)
Date: Sat, 16 Nov 2019 16:16:23 -0800 (PST)
Subject: [pypy-commit] pypy hpy: use the new handles context manager here
Message-ID: <5dd09157.1c69fb81.ca754.139f@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98091:a4e8d4a49fff
Date: 2019-11-17 01:02 +0100
http://bitbucket.org/pypy/pypy/changeset/a4e8d4a49fff/

Log:	use the new handles context manager here

diff --git a/pypy/module/hpy_universal/interp_extfunc.py b/pypy/module/hpy_universal/interp_extfunc.py
--- a/pypy/module/hpy_universal/interp_extfunc.py
+++ b/pypy/module/hpy_universal/interp_extfunc.py
@@ -31,10 +31,9 @@
 
     def call_noargs(self, space):
         state = space.fromcache(State)
-        h_self = handles.new(space, self.w_self)
-        h_result = generic_cpy_call_dont_convert_result(space, self.cfuncptr,
-            state.ctx, h_self, 0)
-        handles.consume(space, h_self)
+        with handles.using(space, self.w_self) as h_self:
+            h_result = generic_cpy_call_dont_convert_result(space, self.cfuncptr,
+                state.ctx, h_self, 0)
         # XXX check for exceptions
         return handles.consume(space, h_result)
 

From pypy.commits at gmail.com  Sat Nov 16 19:16:25 2019
From: pypy.commits at gmail.com (antocuni)
Date: Sat, 16 Nov 2019 16:16:25 -0800 (PST)
Subject: [pypy-commit] pypy hpy: implement METH_O calls;
 test_identify_function passes
Message-ID: <5dd09159.1c69fb81.501f7.0d16@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98092:f51448bff9a4
Date: 2019-11-17 01:09 +0100
http://bitbucket.org/pypy/pypy/changeset/f51448bff9a4/

Log:	implement METH_O calls; test_identify_function passes

diff --git a/pypy/module/hpy_universal/interp_extfunc.py b/pypy/module/hpy_universal/interp_extfunc.py
--- a/pypy/module/hpy_universal/interp_extfunc.py
+++ b/pypy/module/hpy_universal/interp_extfunc.py
@@ -38,7 +38,13 @@
         return handles.consume(space, h_result)
 
     def call_o(self, space, w_arg):
-        raise NotImplementedError("later")
+        state = space.fromcache(State)
+        with handles.using(space, self.w_self) as h_self:
+            with handles.using(space, w_arg) as h_arg:
+                h_result = generic_cpy_call_dont_convert_result(space,
+                                           self.cfuncptr, state.ctx, h_self, h_arg)
+        # XXX check for exceptions
+        return handles.consume(space, h_result)
 
     def call_varargs(self, space, arguments_w):
         raise NotImplementedError("later")
diff --git a/pypy/module/hpy_universal/test/test_basic.py b/pypy/module/hpy_universal/test/test_basic.py
--- a/pypy/module/hpy_universal/test/test_basic.py
+++ b/pypy/module/hpy_universal/test/test_basic.py
@@ -52,3 +52,16 @@
             @INIT
         """)
         assert mod.f() is mod
+
+    def test_identity_function(self):
+        mod = self.make_module("""
+            HPy_FUNCTION(f)
+            static HPy f_impl(HPyContext ctx, HPy self, HPy arg)
+            {
+                return HPy_Dup(ctx, arg);
+            }
+            @EXPORT f METH_O
+            @INIT
+        """)
+        x = object()
+        assert mod.f(x) is x

From pypy.commits at gmail.com  Sat Nov 16 19:16:26 2019
From: pypy.commits at gmail.com (antocuni)
Date: Sat, 16 Nov 2019 16:16:26 -0800 (PST)
Subject: [pypy-commit] pypy hpy: import this test, which already passes :)
Message-ID: <5dd0915a.1c69fb81.b0e75.a8cf@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98093:513f66555585
Date: 2019-11-17 01:14 +0100
http://bitbucket.org/pypy/pypy/changeset/513f66555585/

Log:	import this test, which already passes :)

diff --git a/pypy/module/hpy_universal/test/test_basic.py b/pypy/module/hpy_universal/test/test_basic.py
--- a/pypy/module/hpy_universal/test/test_basic.py
+++ b/pypy/module/hpy_universal/test/test_basic.py
@@ -65,3 +65,27 @@
         """)
         x = object()
         assert mod.f(x) is x
+
+    def test_wrong_number_of_arguments(self):
+        # XXX: this test was manually modified to turn pytest.raises into raises :(
+        mod = self.make_module("""
+            HPy_FUNCTION(f_noargs)
+            static HPy f_noargs_impl(HPyContext ctx, HPy self, HPy args)
+            {
+                return HPyNone_Get(ctx);
+            }
+            HPy_FUNCTION(f_o)
+            static HPy f_o_impl(HPyContext ctx, HPy self, HPy args)
+            {
+                return HPyNone_Get(ctx);
+            }
+            @EXPORT f_noargs METH_NOARGS
+            @EXPORT f_o METH_O
+            @INIT
+        """)
+        with raises(TypeError):
+            mod.f_noargs(1)
+        with raises(TypeError):
+            mod.f_o()
+        with raises(TypeError):
+            mod.f_o(1, 2)

From pypy.commits at gmail.com  Sun Nov 17 05:04:22 2019
From: pypy.commits at gmail.com (arigo)
Date: Sun, 17 Nov 2019 02:04:22 -0800 (PST)
Subject: [pypy-commit] pypy hpy: Reintroduce hpy_universal.load(),
 but keeping .load_from_spec() too
Message-ID: <5dd11b26.1c69fb81.fe85c.56b8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: hpy
Changeset: r98094:bc9e1d24a759
Date: 2019-11-17 11:03 +0100
http://bitbucket.org/pypy/pypy/changeset/bc9e1d24a759/

Log:	Reintroduce hpy_universal.load(), but keeping .load_from_spec() too

diff --git a/pypy/module/hpy_universal/interp_hpy.py b/pypy/module/hpy_universal/interp_hpy.py
--- a/pypy/module/hpy_universal/interp_hpy.py
+++ b/pypy/module/hpy_universal/interp_hpy.py
@@ -63,19 +63,21 @@
     return handles.consume(space, h_module)
 
 def descr_load_from_spec(space, w_spec):
-    # XXX: this looks a lot like cpyext.api.create_extension_module()
+    name = space.text_w(space.getattr(w_spec, space.newtext("name")))
+    origin = space.fsencode_w(space.getattr(w_spec, space.newtext("origin")))
+    return descr_load(space, name, origin)
+
+ at unwrap_spec(name='text', libpath='fsencode')
+def descr_load(space, name, libpath):
     state = space.fromcache(State)
     state.setup()
-    w_name = space.getattr(w_spec, space.newtext("name"))
-    name = space.text_w(w_name)
-    origin = space.text_w(space.getattr(w_spec, space.newtext("origin")))
     try:
-        with rffi.scoped_str2charp(origin) as ll_libname:
+        with rffi.scoped_str2charp(libpath) as ll_libname:
             lib = dlopen(ll_libname, space.sys.dlopenflags)
     except DLOpenError as e:
-        w_path = space.newfilename(origin)
+        w_path = space.newfilename(libpath)
         raise raise_import_error(space,
-            space.newfilename(e.msg), w_name, w_path)
+            space.newfilename(e.msg), space.newtext(name), w_path)
 
     basename = name.split('.')[-1]
     init_name = 'HPyInit_' + basename
@@ -83,8 +85,8 @@
         initptr = dlsym(lib, init_name)
     except KeyError:
         msg = b"function %s not found in library %s" % (
-            init_name, space.utf8_w(space.newfilename(origin)))
-        w_path = space.newfilename(origin)
+            init_name, space.utf8_w(space.newfilename(libpath)))
+        w_path = space.newfilename(libpath)
         raise raise_import_error(
-            space, space.newtext(msg), w_name, w_path)
-    return create_hpy_module(space, name, origin, lib, initptr)
+            space, space.newtext(msg), space.newtext(name), w_path)
+    return create_hpy_module(space, name, libpath, lib, initptr)
diff --git a/pypy/module/hpy_universal/moduledef.py b/pypy/module/hpy_universal/moduledef.py
--- a/pypy/module/hpy_universal/moduledef.py
+++ b/pypy/module/hpy_universal/moduledef.py
@@ -5,5 +5,6 @@
     appleveldefs = {}
 
     interpleveldefs = {
-        'load_from_spec': 'interp_hpy.descr_load_from_spec'
+        'load_from_spec': 'interp_hpy.descr_load_from_spec',
+        'load': 'interp_hpy.descr_load',
     }

From pypy.commits at gmail.com  Sun Nov 17 07:33:29 2019
From: pypy.commits at gmail.com (antocuni)
Date: Sun, 17 Nov 2019 04:33:29 -0800 (PST)
Subject: [pypy-commit] pypy hpy: the JIT doesn't like the casts from FUNCPTR
 to VOIDP, so hide this function from it
Message-ID: <5dd13e19.1c69fb81.690b1.a55e@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98095:69e48dfcbf9c
Date: 2019-11-17 09:25 +0000
http://bitbucket.org/pypy/pypy/changeset/69e48dfcbf9c/

Log:	the JIT doesn't like the casts from FUNCPTR to VOIDP, so hide this
	function from it

diff --git a/pypy/module/hpy_universal/state.py b/pypy/module/hpy_universal/state.py
--- a/pypy/module/hpy_universal/state.py
+++ b/pypy/module/hpy_universal/state.py
@@ -1,6 +1,7 @@
 import os
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.annlowlevel import llhelper
+from rpython.rlib import jit
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib.objectmodel import specialize
 from pypy.module.hpy_universal import llapi
@@ -23,6 +24,7 @@
         self.space = space
         self.ctx = lltype.nullptr(llapi.HPyContext.TO)
 
+    @jit.dont_look_inside
     def setup(self):
         if self.ctx:
             return

From pypy.commits at gmail.com  Sun Nov 17 07:33:31 2019
From: pypy.commits at gmail.com (antocuni)
Date: Sun, 17 Nov 2019 04:33:31 -0800 (PST)
Subject: [pypy-commit] pypy hpy: (antocuni, arigo, ronan): fix translation
Message-ID: <5dd13e1b.1c69fb81.98448.a34a@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98096:3c3b367435c3
Date: 2019-11-17 12:31 +0000
http://bitbucket.org/pypy/pypy/changeset/3c3b367435c3/

Log:	(antocuni, arigo, ronan): fix translation

diff --git a/pypy/module/hpy_universal/llapi.py b/pypy/module/hpy_universal/llapi.py
--- a/pypy/module/hpy_universal/llapi.py
+++ b/pypy/module/hpy_universal/llapi.py
@@ -2,9 +2,28 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 
+# XXX temporary location
+INCLUDE_DIR = os.path.join(os.path.dirname(__file__),
+                           "test", "_vendored", "include")
+
+eci = ExternalCompilationInfo(includes=["universal/hpy.h"],
+                              include_dirs=[INCLUDE_DIR],
+                              post_include_bits=["""
+RPY_EXTERN void *_HPy_GetGlobalCtx(void);
+"""],
+                              separate_module_sources=["""
+
+struct _HPyContext_s hpy_global_ctx;
+void *_HPy_GetGlobalCtx(void)
+{
+    return &hpy_global_ctx;
+}
+
+"""])
+
 
 HPy = lltype.Signed
-HPyContextS = lltype.Struct('dummy_HPyContext_s',
+HPyContextS = rffi.CStruct('_HPyContext_s',
     ('ctx_version', rffi.INT_real),
     ('ctx_Module_Create', rffi.VOIDP),
     ('ctx_None_Get', rffi.VOIDP),
@@ -17,6 +36,7 @@
     ('ctx_FromPyObject', rffi.VOIDP),
     ('ctx_AsPyObject', rffi.VOIDP),
     ('ctx_CallRealFunctionFromTrampoline', rffi.VOIDP),
+    hints={'eci': eci},
 )
 HPyContext = lltype.Ptr(HPyContextS)
 
@@ -35,6 +55,7 @@
     ('ml_meth', _HPyMethodPairFuncPtr),
     ('ml_flags', rffi.INT_real),
     ('ml_doc', rffi.CCHARP),
+    hints={'eci': eci, 'typedef': True},
 )
 
 HPyModuleDef = rffi.CStruct('HPyModuleDef',
@@ -43,6 +64,7 @@
     ('m_doc', rffi.CCHARP),
     ('m_size', lltype.Signed),
     ('m_methods', rffi.CArrayPtr(HPyMethodDef)),
+    hints={'eci': eci, 'typedef': True},
 )
 
 METH_VARARGS  = 0x0001
@@ -53,27 +75,6 @@
 
 # ----------------------------------------------------------------
 
-# XXX temporary location
-INCLUDE_DIR = os.path.join(os.path.dirname(__file__),
-                           "test", "_vendored", "include")
-
-eci = ExternalCompilationInfo(includes=["universal/hpy.h"],
-                              include_dirs=[INCLUDE_DIR],
-                              post_include_bits=["""
-
-RPY_EXTERN void _HPy_FillFunction(int index, void *function);
-RPY_EXTERN void *_HPy_GetGlobalCtx(void);
-"""],
-                              separate_module_sources=["""
-
-struct _HPyContext_s hpy_global_ctx;
-
-void *_HPy_GetGlobalCtx(void)
-{
-    return &hpy_global_ctx;
-}
-
-"""])
 
 _HPy_GetGlobalCtx = rffi.llexternal('_HPy_GetGlobalCtx', [], HPyContext,
                                     compilation_info=eci, _nowrapper=True)
diff --git a/pypy/module/hpy_universal/state.py b/pypy/module/hpy_universal/state.py
--- a/pypy/module/hpy_universal/state.py
+++ b/pypy/module/hpy_universal/state.py
@@ -32,7 +32,7 @@
         self.ctx = llapi._HPy_GetGlobalCtx()
 
         for name in CONTEXT_NAMES:
-            if name != 'ctx_version':
+            if name != 'c_ctx_version':
                 missing_function = make_missing_function(name)
                 funcptr = llhelper(lltype.Ptr(DUMMY_FUNC), missing_function)
                 setattr(self.ctx, name, rffi.cast(rffi.VOIDP, funcptr))
@@ -42,10 +42,10 @@
         space = self.space
         
         funcptr = interp_hpy.HPyModule_Create.get_llhelper(space)
-        self.ctx.ctx_Module_Create = rffi.cast(rffi.VOIDP, funcptr)
+        self.ctx.c_ctx_Module_Create = rffi.cast(rffi.VOIDP, funcptr)
         #
         funcptr = interp_hpy.HPyNone_Get.get_llhelper(space)
-        self.ctx.ctx_None_Get = rffi.cast(rffi.VOIDP, funcptr)
+        self.ctx.c_ctx_None_Get = rffi.cast(rffi.VOIDP, funcptr)
         #
         funcptr = interp_hpy.HPy_Dup.get_llhelper(space)
-        self.ctx.ctx_Dup = rffi.cast(rffi.VOIDP, funcptr)
+        self.ctx.c_ctx_Dup = rffi.cast(rffi.VOIDP, funcptr)

From pypy.commits at gmail.com  Sun Nov 17 09:17:05 2019
From: pypy.commits at gmail.com (arigo)
Date: Sun, 17 Nov 2019 06:17:05 -0800 (PST)
Subject: [pypy-commit] pypy hpy: (antocuni, ronan, arigo)
Message-ID: <5dd15661.1c69fb81.510bc.5c96@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: hpy
Changeset: r98097:44e3a984eb36
Date: 2019-11-17 15:16 +0100
http://bitbucket.org/pypy/pypy/changeset/44e3a984eb36/

Log:	(antocuni, ronan, arigo)

	More general progress

diff --git a/pypy/module/hpy_universal/handles.py b/pypy/module/hpy_universal/handles.py
--- a/pypy/module/hpy_universal/handles.py
+++ b/pypy/module/hpy_universal/handles.py
@@ -20,6 +20,10 @@
         self.handles_w[index] = None
         self.free_list.append(index)
 
+    def deref(self, index):
+        assert index > 0
+        return self.handles_w[index]
+
     def consume(self, index):
         """
         Like close, but also return the w_object which was pointed by the handled
@@ -41,7 +45,11 @@
 def close(space, index):
     mgr = space.fromcache(HandleManager)
     mgr.close(index)
-    
+
+def deref(space, index):
+    mgr = space.fromcache(HandleManager)
+    return mgr.deref(index)
+
 def consume(space, index):
     mgr = space.fromcache(HandleManager)
     return mgr.consume(index)
diff --git a/pypy/module/hpy_universal/interp_extfunc.py b/pypy/module/hpy_universal/interp_extfunc.py
--- a/pypy/module/hpy_universal/interp_extfunc.py
+++ b/pypy/module/hpy_universal/interp_extfunc.py
@@ -47,7 +47,9 @@
         return handles.consume(space, h_result)
 
     def call_varargs(self, space, arguments_w):
-        raise NotImplementedError("later")
+        w_tuple = space.newtuple(arguments_w)
+        # xxx here we just invoke call_o() with the w_tuple
+        return self.call_o(space, w_tuple)
 
     def descr_call(self, space, __args__):
         flags = self.flags
diff --git a/pypy/module/hpy_universal/interp_hpy.py b/pypy/module/hpy_universal/interp_hpy.py
--- a/pypy/module/hpy_universal/interp_hpy.py
+++ b/pypy/module/hpy_universal/interp_hpy.py
@@ -55,6 +55,21 @@
 def HPy_Dup(space, ctx, h):
     return handles.dup(space, h)
 
+ at apifunc([llapi.HPyContext, rffi.LONG], llapi.HPy, error=0)
+def HPyLong_FromLong(space, ctx, value):
+    w_obj = space.newint(rffi.cast(lltype.Signed, value))
+    return handles.new(space, w_obj)
+
+ at apifunc([llapi.HPyContext, llapi.HPy], rffi.LONG, error=0)
+def HPyLong_AsLong(space, ctx, h):
+    w_obj = handles.deref(space, h)
+    #w_obj = space.int(w_obj)     --- XXX write a test for this
+    value = space.int_w(w_obj)
+    result = rffi.cast(rffi.LONG, value)
+    #if rffi.cast(lltype.Signed, result) != value: --- XXX on Windows 64
+    #    ...
+    return result
+
 
 def create_hpy_module(space, name, origin, lib, initfunc):
     state = space.fromcache(State)
diff --git a/pypy/module/hpy_universal/llapi.py b/pypy/module/hpy_universal/llapi.py
--- a/pypy/module/hpy_universal/llapi.py
+++ b/pypy/module/hpy_universal/llapi.py
@@ -30,6 +30,7 @@
     ('ctx_Dup', rffi.VOIDP),
     ('ctx_Close', rffi.VOIDP),
     ('ctx_Long_FromLong', rffi.VOIDP),
+    ('ctx_Long_AsLong', rffi.VOIDP),
     ('ctx_Arg_ParseTuple', rffi.VOIDP),
     ('ctx_Number_Add', rffi.VOIDP),
     ('ctx_Unicode_FromString', rffi.VOIDP),
diff --git a/pypy/module/hpy_universal/state.py b/pypy/module/hpy_universal/state.py
--- a/pypy/module/hpy_universal/state.py
+++ b/pypy/module/hpy_universal/state.py
@@ -49,3 +49,9 @@
         #
         funcptr = interp_hpy.HPy_Dup.get_llhelper(space)
         self.ctx.c_ctx_Dup = rffi.cast(rffi.VOIDP, funcptr)
+        #
+        funcptr = interp_hpy.HPyLong_FromLong.get_llhelper(space)
+        self.ctx.c_ctx_Long_FromLong = rffi.cast(rffi.VOIDP, funcptr)
+        #
+        funcptr = interp_hpy.HPyLong_AsLong.get_llhelper(space)
+        self.ctx.c_ctx_Long_AsLong = rffi.cast(rffi.VOIDP, funcptr)
diff --git a/pypy/module/hpy_universal/test/_vendored/include/cpython/hpy.h b/pypy/module/hpy_universal/test/_vendored/include/cpython/hpy.h
--- a/pypy/module/hpy_universal/test/_vendored/include/cpython/hpy.h
+++ b/pypy/module/hpy_universal/test/_vendored/include/cpython/hpy.h
@@ -111,6 +111,12 @@
     return _py2h(PyLong_FromLong(v));
 }
 
+HPyAPI_FUNC(long)
+HPyLong_AsLong(HPyContext ctx, HPy h)
+{
+    return PyLong_AsLong(_h2py(h));
+}
+
 HPyAPI_FUNC(HPy)
 HPyNumber_Add(HPyContext ctx, HPy x, HPy y)
 {
diff --git a/pypy/module/hpy_universal/test/_vendored/include/universal/autogen_ctx.h b/pypy/module/hpy_universal/test/_vendored/include/universal/autogen_ctx.h
--- a/pypy/module/hpy_universal/test/_vendored/include/universal/autogen_ctx.h
+++ b/pypy/module/hpy_universal/test/_vendored/include/universal/autogen_ctx.h
@@ -15,6 +15,7 @@
     HPy (*ctx_Dup)(HPyContext ctx, HPy h);
     void (*ctx_Close)(HPyContext ctx, HPy h);
     HPy (*ctx_Long_FromLong)(HPyContext ctx, long value);
+    long (*ctx_Long_AsLong)(HPyContext ctx, HPy h);
     int (*ctx_Arg_ParseTuple)(HPyContext ctx, HPy args, const char *fmt, va_list _vl);
     HPy (*ctx_Number_Add)(HPyContext ctx, HPy x, HPy y);
     HPy (*ctx_Unicode_FromString)(HPyContext ctx, const char *utf8);
diff --git a/pypy/module/hpy_universal/test/_vendored/include/universal/autogen_func.h b/pypy/module/hpy_universal/test/_vendored/include/universal/autogen_func.h
--- a/pypy/module/hpy_universal/test/_vendored/include/universal/autogen_func.h
+++ b/pypy/module/hpy_universal/test/_vendored/include/universal/autogen_func.h
@@ -28,6 +28,10 @@
      return ctx->ctx_Long_FromLong ( ctx, value ); 
 }
 
+static inline long HPyLong_AsLong(HPyContext ctx, HPy h) {
+     return ctx->ctx_Long_AsLong ( ctx, h ); 
+}
+
 static inline int HPyArg_ParseTuple(HPyContext ctx, HPy args, const char *fmt, ...) {
      va_list _vl; va_start(_vl, fmt); int _res = ctx->ctx_Arg_ParseTuple ( ctx, args, fmt, _vl ); va_end(_vl); return _res; 
 }
diff --git a/pypy/module/hpy_universal/test/_vendored/include/universal/hpy.h b/pypy/module/hpy_universal/test/_vendored/include/universal/hpy.h
--- a/pypy/module/hpy_universal/test/_vendored/include/universal/hpy.h
+++ b/pypy/module/hpy_universal/test/_vendored/include/universal/hpy.h
@@ -39,8 +39,9 @@
     HPyMethodDef *m_methods;
 } HPyModuleDef;
 
+#define _HPy_HIDDEN   __attribute__((visibility("hidden")))
 #define HPy_MODINIT(modname)                                   \
-    HPyContext _ctx_for_trampolines;                           \
+    _HPy_HIDDEN HPyContext _ctx_for_trampolines;               \
     static HPy init_##modname##_impl(HPyContext ctx);          \
     HPy HPyInit_##modname(HPyContext ctx)                      \
     {                                                          \
diff --git a/pypy/module/hpy_universal/test/test_basic.py b/pypy/module/hpy_universal/test/test_basic.py
--- a/pypy/module/hpy_universal/test/test_basic.py
+++ b/pypy/module/hpy_universal/test/test_basic.py
@@ -66,6 +66,19 @@
         x = object()
         assert mod.f(x) is x
 
+    def test_long_aslong(self):
+        mod = self.make_module("""
+            HPy_FUNCTION(f)
+            static HPy f_impl(HPyContext ctx, HPy self, HPy arg)
+            {
+                long a = HPyLong_AsLong(ctx, arg);
+                return HPyLong_FromLong(ctx, a * 2);
+            }
+            @EXPORT f METH_O
+            @INIT
+        """)
+        assert mod.f(45) == 90
+
     def test_wrong_number_of_arguments(self):
         # XXX: this test was manually modified to turn pytest.raises into raises :(
         mod = self.make_module("""
diff --git a/pypy/module/hpy_universal/test/test_handles.py b/pypy/module/hpy_universal/test/test_handles.py
--- a/pypy/module/hpy_universal/test/test_handles.py
+++ b/pypy/module/hpy_universal/test/test_handles.py
@@ -3,6 +3,11 @@
 
 class TestHandleManager(object):
 
+    def test_first_handle_is_not_zero(self):
+        mgr = HandleManager(None)
+        h = mgr.new('hello')
+        assert h > 0
+
     def test_new(self):
         mgr = HandleManager(None)
         h = mgr.new('hello')
@@ -14,6 +19,12 @@
         assert mgr.close(h) is None
         assert mgr.handles_w[h] is None
 
+    def test_deref(self):
+        mgr = HandleManager(None)
+        h = mgr.new('hello')
+        assert mgr.deref(h) == 'hello'     # 'hello' is a fake W_Root
+        assert mgr.deref(h) == 'hello'
+
     def test_consume(self):
         mgr = HandleManager(None)
         h = mgr.new('hello')

From pypy.commits at gmail.com  Sun Nov 17 12:41:23 2019
From: pypy.commits at gmail.com (arigo)
Date: Sun, 17 Nov 2019 09:41:23 -0800 (PST)
Subject: [pypy-commit] pypy default: Revert 18443d3a74d5: the branch
 int-test-is-zero is maybe a nice idea
Message-ID: <5dd18643.1c69fb81.2eb99.4f28@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r98098:37894e68890d
Date: 2019-11-17 18:40 +0100
http://bitbucket.org/pypy/pypy/changeset/37894e68890d/

Log:	Revert 18443d3a74d5: the branch int-test-is-zero is maybe a nice
	idea but it doesn't really work. E.g. when we write "if a & 3:" at
	app-level then the value "a & 3" is anyway passed around as
	failargs. So this branch makes the CPU compute twice this value,
	basically.

	This is an almost-complete-revert: I'm keeping the new TEST_xx
	instructions in rx86.py because the next time they're needed we'll
	already have them.

diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -1,5 +1,4 @@
 import py, sys
-import platform
 from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC
 
 
@@ -421,23 +420,3 @@
         # the following assertion fails if the loop was cancelled due
         # to "abort: vable escape"
         assert len(loops) == 1
-
-    def test_bit_check(self):
-        if not platform.machine().startswith('x86'):
-            py.test.skip("only x86 supports int_test_instructions for now")
-
-        def main(n):
-            x = 0
-            while n:
-                y = bool(n & 7)    # ID: bitcheck
-                x += y
-                n -= 1
-
-        log = self.run(main, [300])
-        loop, = log.loops_by_id("bitcheck")
-        assert loop.match_by_id("bitcheck", """
-            guard_not_invalidated?
-            i11 = int_and(i7, 7)    # not used
-            i12 = int_test_is_true(i7, 7)
-            guard_true(i12, descr=...)
-        """)
diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -333,7 +333,6 @@
     vector_ext.enable(16, accum=True)
     vector_ext.setup_once = lambda asm: asm
     load_supported_factors = (1,2,4,8)
-    supports_int_test_instructions = True
     assembler = None
 
     def __init__(self, rtyper, stats=None, *ignored_args, **kwds):
diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py
--- a/rpython/jit/backend/model.py
+++ b/rpython/jit/backend/model.py
@@ -20,7 +20,6 @@
     supports_singlefloats = False
     supports_guard_gc_type = False
     supports_load_effective_address = False
-    supports_int_test_instructions = False
 
     propagate_exception_descr = None
 
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -1491,8 +1491,6 @@
             (rop.INT_NE, lambda x, y: x != y),
             (rop.INT_GT, lambda x, y: x > y),
             (rop.INT_GE, lambda x, y: x >= y),
-            (rop.INT_TEST_IS_ZERO, lambda x, y: (x & y) == 0),
-            (rop.INT_TEST_IS_TRUE, lambda x, y: (x & y) != 0),
             ]:
             for opguard, guard_case in [
                 (rop.GUARD_FALSE, False),
diff --git a/rpython/jit/backend/test/test_random.py b/rpython/jit/backend/test/test_random.py
--- a/rpython/jit/backend/test/test_random.py
+++ b/rpython/jit/backend/test/test_random.py
@@ -546,8 +546,6 @@
             rop.UINT_LE,
             rop.UINT_GT,
             rop.UINT_GE,
-            rop.INT_TEST_IS_ZERO,
-            rop.INT_TEST_IS_TRUE,
             ]:
     OPERATIONS.append(BinaryOperation(_op, boolres=True))
 
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1281,17 +1281,6 @@
                 self.flush_cc(cond, result_loc)
         return genop_cmp
 
-    def _testop(cond):
-        cond = rx86.Conditions[cond]
-        #
-        def genop_test(self, op, arglocs, result_loc):
-            if arglocs[1].is_stack() or isinstance(arglocs[0], ImmedLoc):
-                self.mc.TEST(arglocs[1], arglocs[0])
-            else:
-                self.mc.TEST(arglocs[0], arglocs[1])
-            self.flush_cc(cond, result_loc)
-        return genop_test
-
     def _if_parity_clear_zero_and_carry(self):
         jnp_location = self.mc.emit_forward_jump('NP')
         # CMP EBP, 0: as EBP cannot be null here, that operation should
@@ -1412,9 +1401,6 @@
     genop_float_gt = _cmpop_float("A", "B")
     genop_float_ge = _cmpop_float("AE","BE")
 
-    genop_int_test_is_zero = _testop("Z")
-    genop_int_test_is_true = _testop("NZ")
-
     def genop_math_sqrt(self, op, arglocs, resloc):
         self.mc.SQRTSD(arglocs[0], resloc)
 
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -644,8 +644,6 @@
     consider_uint_ge = _consider_compop
     consider_ptr_eq = consider_instance_ptr_eq = _consider_compop
     consider_ptr_ne = consider_instance_ptr_ne = _consider_compop
-    consider_int_test_is_zero = _consider_compop
-    consider_int_test_is_true = _consider_compop
 
     def _consider_float_op(self, op):
         loc1 = self.xrm.loc(op.getarg(1))
diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py
--- a/rpython/jit/backend/x86/runner.py
+++ b/rpython/jit/backend/x86/runner.py
@@ -17,7 +17,6 @@
     supports_floats = True
     supports_singlefloats = True
     supports_load_effective_address = True
-    supports_int_test_instructions = True
 
     dont_keepalive_stuff = False # for tests
     with_threads = False
diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -542,12 +542,6 @@
     @arguments("i", "i", returns="i")
     def bhimpl_int_signext(a, b):
         return int_signext(a, b)
-    @arguments("i", "i", returns="i")
-    def bhimpl_int_test_is_zero(a, b):
-        return (a & b) == 0
-    @arguments("i", "i", returns="i")
-    def bhimpl_int_test_is_true(a, b):
-        return (a & b) != 0
 
     @arguments("i", "i", returns="i")
     def bhimpl_uint_lt(a, b):
diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -442,8 +442,6 @@
                          rop.GC_STORE_INDEXED,
                          rop.LOAD_FROM_GC_TABLE,
                          rop.LOAD_EFFECTIVE_ADDRESS,
-                         rop.INT_TEST_IS_ZERO,
-                         rop.INT_TEST_IS_TRUE,
                          ):      # list of opcodes never executed by pyjitpl
                 continue
             if rop._VEC_PURE_FIRST <= value <= rop._VEC_PURE_LAST:
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -646,16 +646,6 @@
         elif info == INFO_NULL:
             self.make_constant_int(op, not expect_nonnull)
         else:
-            if self.optimizer.cpu.supports_int_test_instructions:
-                box = get_box_replacement(box)
-                box1 = self.optimizer.as_operation(box)
-                if box1 is not None and box1.getopnum() == rop.INT_AND:
-                    if expect_nonnull:
-                        opnum = rop.INT_TEST_IS_TRUE
-                    else:
-                        opnum = rop.INT_TEST_IS_ZERO
-                    args = [box1.getarg(0), box1.getarg(1)]
-                    op = self.replace_op_with(op, opnum, args=args)
             return self.emit(op)
 
     def optimize_INT_IS_TRUE(self, op):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -5790,33 +5790,3 @@
         i57 = int_or(i51, i52)
         """
         self.optimize_loop(ops, expected)
-
-    def test_int_test_is_zero(self):
-        ops = """
-        [i1, i2]
-        i51 = int_and(i1, i2)
-        i52 = int_is_zero(i51)
-        guard_true(i52) []
-        """
-        expected = """
-        [i1, i2]
-        i51 = int_and(i1, i2)      # likely dead instruction
-        i52 = int_test_is_zero(i1, i2)
-        guard_true(i52) []
-        """
-        self.optimize_loop(ops, expected)
-
-    def test_int_test_is_true(self):
-        ops = """
-        [i1, i2]
-        i51 = int_and(i1, i2)
-        i52 = int_is_true(i51)
-        guard_true(i52) []
-        """
-        expected = """
-        [i1, i2]
-        i51 = int_and(i1, i2)      # likely dead instruction
-        i52 = int_test_is_true(i1, i2)
-        guard_true(i52) []
-        """
-        self.optimize_loop(ops, expected)
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -1036,8 +1036,6 @@
     'INT_NEG/1/i',
     'INT_INVERT/1/i',
     'INT_FORCE_GE_ZERO/1/i',
-    'INT_TEST_IS_ZERO/2b/i',
-    'INT_TEST_IS_TRUE/2b/i',
     #
     'SAME_AS/1/ifr',      # gets a Const or a Box, turns it into another Box
     'CAST_PTR_TO_INT/1/i',
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4832,11 +4832,3 @@
         res2 = self.interp_operations(f, [6])
         assert res1 == res2
         self.check_operations_history(guard_class=1, record_exact_class=0)
-
-    def test_int_test_instructions(self):
-        def f(x, y):
-            if (x & 7) == 0 and (y & 7) != 0:
-                return 1
-            return 0
-        res = self.interp_operations(f, [24, 25])
-        assert res == 1
diff --git a/rpython/jit/metainterp/test/test_executor.py b/rpython/jit/metainterp/test/test_executor.py
--- a/rpython/jit/metainterp/test/test_executor.py
+++ b/rpython/jit/metainterp/test/test_executor.py
@@ -187,8 +187,6 @@
         (rop.UINT_LE, lambda x, y: r_uint(x) <= r_uint(y)),
         (rop.UINT_GT, lambda x, y: r_uint(x) >  r_uint(y)),
         (rop.UINT_GE, lambda x, y: r_uint(x) >= r_uint(y)),
-        (rop.INT_TEST_IS_ZERO, lambda x, y: (x & y) == 0),
-        (rop.INT_TEST_IS_TRUE, lambda x, y: (x & y) != 0),
         ]:
         for i in range(20):
             x = pick()

From pypy.commits at gmail.com  Sun Nov 17 13:09:24 2019
From: pypy.commits at gmail.com (arigo)
Date: Sun, 17 Nov 2019 10:09:24 -0800 (PST)
Subject: [pypy-commit] pypy hpy: HPy_Close(), HPyNumber_Add()
Message-ID: <5dd18cd4.1c69fb81.322d4.a66d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: hpy
Changeset: r98099:25a647d7d077
Date: 2019-11-17 19:08 +0100
http://bitbucket.org/pypy/pypy/changeset/25a647d7d077/

Log:	HPy_Close(), HPyNumber_Add()

diff --git a/pypy/module/hpy_universal/interp_hpy.py b/pypy/module/hpy_universal/interp_hpy.py
--- a/pypy/module/hpy_universal/interp_hpy.py
+++ b/pypy/module/hpy_universal/interp_hpy.py
@@ -55,6 +55,10 @@
 def HPy_Dup(space, ctx, h):
     return handles.dup(space, h)
 
+ at apifunc([llapi.HPyContext, llapi.HPy], lltype.Void, error=None)
+def HPy_Close(space, ctx, h):
+    handles.close(space, h)
+
 @apifunc([llapi.HPyContext, rffi.LONG], llapi.HPy, error=0)
 def HPyLong_FromLong(space, ctx, value):
     w_obj = space.newint(rffi.cast(lltype.Signed, value))
@@ -70,6 +74,13 @@
     #    ...
     return result
 
+ at apifunc([llapi.HPyContext, llapi.HPy, llapi.HPy], llapi.HPy, error=0)
+def HPyNumber_Add(space, ctx, h1, h2):
+    w_obj1 = handles.deref(space, h1)
+    w_obj2 = handles.deref(space, h2)
+    w_result = space.add(w_obj1, w_obj2)
+    return handles.new(space, w_result)
+
 
 def create_hpy_module(space, name, origin, lib, initfunc):
     state = space.fromcache(State)
diff --git a/pypy/module/hpy_universal/state.py b/pypy/module/hpy_universal/state.py
--- a/pypy/module/hpy_universal/state.py
+++ b/pypy/module/hpy_universal/state.py
@@ -50,8 +50,14 @@
         funcptr = interp_hpy.HPy_Dup.get_llhelper(space)
         self.ctx.c_ctx_Dup = rffi.cast(rffi.VOIDP, funcptr)
         #
+        funcptr = interp_hpy.HPy_Close.get_llhelper(space)
+        self.ctx.c_ctx_Close = rffi.cast(rffi.VOIDP, funcptr)
+        #
         funcptr = interp_hpy.HPyLong_FromLong.get_llhelper(space)
         self.ctx.c_ctx_Long_FromLong = rffi.cast(rffi.VOIDP, funcptr)
         #
         funcptr = interp_hpy.HPyLong_AsLong.get_llhelper(space)
         self.ctx.c_ctx_Long_AsLong = rffi.cast(rffi.VOIDP, funcptr)
+        #
+        funcptr = interp_hpy.HPyNumber_Add.get_llhelper(space)
+        self.ctx.c_ctx_Number_Add = rffi.cast(rffi.VOIDP, funcptr)
diff --git a/pypy/module/hpy_universal/test/test_basic.py b/pypy/module/hpy_universal/test/test_basic.py
--- a/pypy/module/hpy_universal/test/test_basic.py
+++ b/pypy/module/hpy_universal/test/test_basic.py
@@ -102,3 +102,20 @@
             mod.f_o()
         with raises(TypeError):
             mod.f_o(1, 2)
+
+    def test_close(self):
+        mod = self.make_module("""
+            HPy_FUNCTION(f)
+            static HPy f_impl(HPyContext ctx, HPy self, HPy arg)
+            {
+                HPy one = HPyLong_FromLong(ctx, 1);
+                if (HPy_IsNull(one))
+                    return HPy_NULL;
+                HPy res = HPyNumber_Add(ctx, arg, one);
+                HPy_Close(ctx, one);
+                return res;
+            }
+            @EXPORT f METH_O
+            @INIT
+        """)
+        assert mod.f(41.5) == 42.5

From pypy.commits at gmail.com  Sun Nov 17 13:29:26 2019
From: pypy.commits at gmail.com (arigo)
Date: Sun, 17 Nov 2019 10:29:26 -0800 (PST)
Subject: [pypy-commit] pypy hpy: HPyUnicode_FromString()
Message-ID: <5dd19186.1c69fb81.d01ba.cb63@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: hpy
Changeset: r98100:6009d087288b
Date: 2019-11-17 19:28 +0100
http://bitbucket.org/pypy/pypy/changeset/6009d087288b/

Log:	HPyUnicode_FromString()

diff --git a/pypy/module/hpy_universal/interp_hpy.py b/pypy/module/hpy_universal/interp_hpy.py
--- a/pypy/module/hpy_universal/interp_hpy.py
+++ b/pypy/module/hpy_universal/interp_hpy.py
@@ -2,6 +2,7 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib.rdynload import dlopen, dlsym, DLOpenError
 from rpython.rlib.objectmodel import specialize
+from rpython.rlib import rutf8
 
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.error import raise_import_error
@@ -81,6 +82,16 @@
     w_result = space.add(w_obj1, w_obj2)
     return handles.new(space, w_result)
 
+ at apifunc([llapi.HPyContext, rffi.CCHARP], llapi.HPy, error=0)
+def HPyUnicode_FromString(space, ctx, utf8):
+    s = rffi.charp2str(utf8)
+    try:
+        length = rutf8.check_utf8(s, allow_surrogates=False)
+    except rutf8.CheckError:
+        raise   # XXX do something
+    w_obj = space.newtext(s, length)
+    return handles.new(space, w_obj)
+
 
 def create_hpy_module(space, name, origin, lib, initfunc):
     state = space.fromcache(State)
diff --git a/pypy/module/hpy_universal/state.py b/pypy/module/hpy_universal/state.py
--- a/pypy/module/hpy_universal/state.py
+++ b/pypy/module/hpy_universal/state.py
@@ -61,3 +61,6 @@
         #
         funcptr = interp_hpy.HPyNumber_Add.get_llhelper(space)
         self.ctx.c_ctx_Number_Add = rffi.cast(rffi.VOIDP, funcptr)
+        #
+        funcptr = interp_hpy.HPyUnicode_FromString.get_llhelper(space)
+        self.ctx.c_ctx_Unicode_FromString = rffi.cast(rffi.VOIDP, funcptr)
diff --git a/pypy/module/hpy_universal/test/test_basic.py b/pypy/module/hpy_universal/test/test_basic.py
--- a/pypy/module/hpy_universal/test/test_basic.py
+++ b/pypy/module/hpy_universal/test/test_basic.py
@@ -119,3 +119,15 @@
             @INIT
         """)
         assert mod.f(41.5) == 42.5
+
+    def test_string(self):
+        mod = self.make_module("""
+            HPy_FUNCTION(f)
+            static HPy f_impl(HPyContext ctx, HPy self, HPy args)
+            {
+                return HPyUnicode_FromString(ctx, "foobar");
+            }
+            @EXPORT f METH_NOARGS
+            @INIT
+        """)
+        assert mod.f() == "foobar"

From pypy.commits at gmail.com  Sun Nov 17 19:33:30 2019
From: pypy.commits at gmail.com (antocuni)
Date: Sun, 17 Nov 2019 16:33:30 -0800 (PST)
Subject: [pypy-commit] pypy hpy: add a script to update the _vendored
 directory from the hpy repo
Message-ID: <5dd1e6da.1c69fb81.94a41.0ab6@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98101:ff086c2fef83
Date: 2019-11-18 01:32 +0100
http://bitbucket.org/pypy/pypy/changeset/ff086c2fef83/

Log:	add a script to update the _vendored directory from the hpy repo

diff --git a/pypy/module/hpy_universal/update_vendored.sh b/pypy/module/hpy_universal/update_vendored.sh
new file mode 100755
--- /dev/null
+++ b/pypy/module/hpy_universal/update_vendored.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -e
+
+if [ "$#" -ne 1 ]; then
+    echo "Usage: $0 /path/to/hpy"
+    exit 1
+fi
+
+DIR=$(dirname $0)
+VENDORED=$DIR/test/_vendored
+HPY=$1
+
+echo "status of the repo $HPY:"
+git -C "$HPY" --no-pager log --oneline -n 1
+git -C "$HPY" describe --abbrev --always --dirty
+
+cp -R "$HPY"/hpy-api/hpy_devel/include/* "$VENDORED/include"
+#cp $HPY/test/support.py $VENDORED/
+

From pypy.commits at gmail.com  Mon Nov 18 06:28:36 2019
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 18 Nov 2019 03:28:36 -0800 (PST)
Subject: [pypy-commit] pypy hpy: move the _vendored directory one level up
 and reorganize things until tests pass again
Message-ID: <5dd28064.1c69fb81.b5677.7aec@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98102:c385bc205649
Date: 2019-11-18 09:17 +0100
http://bitbucket.org/pypy/pypy/changeset/c385bc205649/

Log:	move the _vendored directory one level up and reorganize things
	until tests pass again

diff --git a/pypy/module/hpy_universal/test/_vendored/__init__.py b/pypy/module/hpy_universal/_vendored/__init__.py
rename from pypy/module/hpy_universal/test/_vendored/__init__.py
rename to pypy/module/hpy_universal/_vendored/__init__.py
diff --git a/pypy/module/hpy_universal/test/_vendored/include/cpython/hpy.h b/pypy/module/hpy_universal/_vendored/include/cpython/hpy.h
rename from pypy/module/hpy_universal/test/_vendored/include/cpython/hpy.h
rename to pypy/module/hpy_universal/_vendored/include/cpython/hpy.h
diff --git a/pypy/module/hpy_universal/test/_vendored/include/hpy.h b/pypy/module/hpy_universal/_vendored/include/hpy.h
rename from pypy/module/hpy_universal/test/_vendored/include/hpy.h
rename to pypy/module/hpy_universal/_vendored/include/hpy.h
diff --git a/pypy/module/hpy_universal/test/_vendored/include/universal/autogen_ctx.h b/pypy/module/hpy_universal/_vendored/include/universal/autogen_ctx.h
rename from pypy/module/hpy_universal/test/_vendored/include/universal/autogen_ctx.h
rename to pypy/module/hpy_universal/_vendored/include/universal/autogen_ctx.h
diff --git a/pypy/module/hpy_universal/test/_vendored/include/universal/autogen_func.h b/pypy/module/hpy_universal/_vendored/include/universal/autogen_func.h
rename from pypy/module/hpy_universal/test/_vendored/include/universal/autogen_func.h
rename to pypy/module/hpy_universal/_vendored/include/universal/autogen_func.h
diff --git a/pypy/module/hpy_universal/test/_vendored/include/universal/hpy.h b/pypy/module/hpy_universal/_vendored/include/universal/hpy.h
rename from pypy/module/hpy_universal/test/_vendored/include/universal/hpy.h
rename to pypy/module/hpy_universal/_vendored/include/universal/hpy.h
diff --git a/pypy/module/hpy_universal/_vendored/test/__init__.py b/pypy/module/hpy_universal/_vendored/test/__init__.py
new file mode 100644
diff --git a/pypy/module/hpy_universal/test/_vendored/support.py b/pypy/module/hpy_universal/_vendored/test/support.py
rename from pypy/module/hpy_universal/test/_vendored/support.py
rename to pypy/module/hpy_universal/_vendored/test/support.py
diff --git a/pypy/module/hpy_universal/llapi.py b/pypy/module/hpy_universal/llapi.py
--- a/pypy/module/hpy_universal/llapi.py
+++ b/pypy/module/hpy_universal/llapi.py
@@ -1,11 +1,9 @@
-import os
+import py
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
+from pypy.module.hpy_universal import _vendored
 
-# XXX temporary location
-INCLUDE_DIR = os.path.join(os.path.dirname(__file__),
-                           "test", "_vendored", "include")
-
+INCLUDE_DIR = str(py.path.local(_vendored.__file__).dirpath().join('include'))
 eci = ExternalCompilationInfo(includes=["universal/hpy.h"],
                               include_dirs=[INCLUDE_DIR],
                               post_include_bits=["""
diff --git a/pypy/module/hpy_universal/test/support.py b/pypy/module/hpy_universal/test/support.py
--- a/pypy/module/hpy_universal/test/support.py
+++ b/pypy/module/hpy_universal/test/support.py
@@ -1,11 +1,11 @@
 import py
 import pytest
+from rpython.tool.udir import udir
 from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.module.hpy_universal.llapi import INCLUDE_DIR
+from pypy.module.hpy_universal._vendored.test import support as _support
 
-from rpython.tool.udir import udir
-from ._vendored import support as _support
 
-INCLUDE_DIR = str(py.path.local(__file__).dirpath().join('_vendored/include'))
 
 class ExtensionCompiler(object):
     def __init__(self, base_dir):

From pypy.commits at gmail.com  Mon Nov 18 06:28:38 2019
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 18 Nov 2019 03:28:38 -0800 (PST)
Subject: [pypy-commit] pypy hpy: update the script with the new location
Message-ID: <5dd28066.1c69fb81.63b8a.e5b0@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98103:5b07516b0134
Date: 2019-11-18 10:56 +0100
http://bitbucket.org/pypy/pypy/changeset/5b07516b0134/

Log:	update the script with the new location

diff --git a/pypy/module/hpy_universal/update_vendored.sh b/pypy/module/hpy_universal/update_vendored.sh
--- a/pypy/module/hpy_universal/update_vendored.sh
+++ b/pypy/module/hpy_universal/update_vendored.sh
@@ -8,7 +8,7 @@
 fi
 
 DIR=$(dirname $0)
-VENDORED=$DIR/test/_vendored
+VENDORED=$DIR/_vendored
 HPY=$1
 
 echo "status of the repo $HPY:"
@@ -16,5 +16,5 @@
 git -C "$HPY" describe --abbrev --always --dirty
 
 cp -R "$HPY"/hpy-api/hpy_devel/include/* "$VENDORED/include"
-#cp $HPY/test/support.py $VENDORED/
+cp -R "$HPY"/test/* "$VENDORED/test"
 

From pypy.commits at gmail.com  Mon Nov 18 06:28:40 2019
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 18 Nov 2019 03:28:40 -0800 (PST)
Subject: [pypy-commit] pypy hpy: make the script nicer
Message-ID: <5dd28068.1c69fb81.5db41.bec8@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98104:114766a84394
Date: 2019-11-18 11:08 +0100
http://bitbucket.org/pypy/pypy/changeset/114766a84394/

Log:	make the script nicer

diff --git a/pypy/module/hpy_universal/update_vendored.sh b/pypy/module/hpy_universal/update_vendored.sh
--- a/pypy/module/hpy_universal/update_vendored.sh
+++ b/pypy/module/hpy_universal/update_vendored.sh
@@ -11,10 +11,14 @@
 VENDORED=$DIR/_vendored
 HPY=$1
 
-echo "status of the repo $HPY:"
+echo "GIT status of $HPY"
 git -C "$HPY" --no-pager log --oneline -n 1
-git -C "$HPY" describe --abbrev --always --dirty
+git -C "$HPY" --no-pager diff --stat
 
-cp -R "$HPY"/hpy-api/hpy_devel/include/* "$VENDORED/include"
+#cp -R "$HPY"/hpy-api/hpy_devel/include/* "$VENDORED/include"
 cp -R "$HPY"/test/* "$VENDORED/test"
 
+echo
+echo
+echo "HG status of pypy"
+hg st $DIR

From pypy.commits at gmail.com  Mon Nov 18 06:28:41 2019
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 18 Nov 2019 03:28:41 -0800 (PST)
Subject: [pypy-commit] pypy hpy: update support.py to the git hpy revision
 48d7fda
Message-ID: <5dd28069.1c69fb81.3c704.df89@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98105:e24f375edf83
Date: 2019-11-18 11:10 +0100
http://bitbucket.org/pypy/pypy/changeset/e24f375edf83/

Log:	update support.py to the git hpy revision 48d7fda

diff --git a/pypy/module/hpy_universal/_vendored/test/support.py b/pypy/module/hpy_universal/_vendored/test/support.py
--- a/pypy/module/hpy_universal/_vendored/test/support.py
+++ b/pypy/module/hpy_universal/_vendored/test/support.py
@@ -1,12 +1,6 @@
 import os, sys
 import pytest
 import re
-#import importlib.util
-#from importlib.machinery import ExtensionFileLoader
-
-THIS_DIR = os.path.dirname(__file__)
-INCLUDE_DIR = os.path.join(THIS_DIR, '../hpy-api/include')
-
 
 r_marker_init = re.compile(r"\s*@INIT\s*$")
 r_marker_export = re.compile(r"\s*@EXPORT\s+(\w+)\s+(METH_\w+)\s*$")
@@ -61,34 +55,61 @@
     return '\n'.join(expanded_lines)
 
 
-#class HPyLoader(ExtensionFileLoader):
-#    def create_module(self, spec):
-#        import hpy_universal
-#        return hpy_universal.load(spec.origin, "HPyInit_" + spec.name)
+class Spec(object):
+    def __init__(self, name, origin):
+        self.name = name
+        self.origin = origin
+
 
 class ExtensionCompiler:
-    def __init__(self, tmpdir, abimode):
+    def __init__(self, tmpdir, abimode, include_dir):
         self.tmpdir = tmpdir
         self.abimode = abimode
+        self.include_dir = include_dir
+        self.universal_mode = self.abimode == 'universal'
 
-    def make_module(self, source_template, name):
-        universal_mode = self.abimode == 'universal'
+    def compile_module(self, source_template, name):
+        """
+        Create and compile a HPy module from the template
+        """
         source = expand_template(source_template, name)
         filename = self.tmpdir.join(name + '.c')
         filename.write(source)
         #
-        ext = get_extension(str(filename), name, include_dirs=[INCLUDE_DIR],
+        ext = get_extension(str(filename), name,
+                            include_dirs=[self.include_dir],
                             extra_compile_args=['-Wfatal-errors'])
         so_filename = c_compile(str(self.tmpdir), ext, compiler_verbose=False,
-                                universal_mode=universal_mode)
-        #
-        if universal_mode:
-            loader = HPyLoader(name, so_filename)
-            spec = importlib.util.spec_from_loader(name, loader)
+                                universal_mode=self.universal_mode)
+        return so_filename
+
+    def make_module(self, source_template, name):
+        """
+        Compile&load a modulo into memory. This is NOT a proper import: e.g. the module
+        is not put into sys.modules
+        """
+        so_filename = self.compile_module(source_template, name)
+        if self.universal_mode:
+            return self.load_universal_module(name, so_filename)
         else:
-            spec = importlib.util.spec_from_file_location(name, so_filename)
+            return self.load_cython_module(name, so_filename)
+
+    def load_universal_module(self, name, so_filename):
+        assert self.abimode == 'universal'
+        import hpy_universal
+        spec = Spec(name, so_filename)
+        return hpy_universal.load_from_spec(spec)
+
+    def load_cython_module(self, name, so_filename):
+        assert self.abimode == 'cpython'
+        # we've got a normal CPython module compiled with the CPython API/ABI,
+        # let's load it normally. It is important to do the imports only here,
+        # because this file will be imported also by PyPy tests which runs on
+        # Python2
+        import importlib.util
+        from importlib.machinery import ExtensionFileLoader
+        spec = importlib.util.spec_from_file_location(name, so_filename)
         module = importlib.util.module_from_spec(spec)
-        sys.modules[name] = module
         spec.loader.exec_module(module)
         return module
 
@@ -97,8 +118,12 @@
 class HPyTest:
     @pytest.fixture()
     def initargs(self, compiler):
+        # compiler is a fixture defined in conftest
         self.compiler = compiler
 
+    def compile_module(self, source_template, name):
+        return self.compiler.compile_module(source_template, name)
+
     def make_module(self, source_template, name='mytest'):
         return self.compiler.make_module(source_template, name)
 

From pypy.commits at gmail.com  Mon Nov 18 06:28:43 2019
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 18 Nov 2019 03:28:43 -0800 (PST)
Subject: [pypy-commit] pypy hpy: we no longer need the FakeSpec,
 we can call hpy_universal.load directly now
Message-ID: <5dd2806b.1c69fb81.174b6.51d2@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98106:2f6fc739e150
Date: 2019-11-18 11:13 +0100
http://bitbucket.org/pypy/pypy/changeset/2f6fc739e150/

Log:	we no longer need the FakeSpec, we can call hpy_universal.load
	directly now

diff --git a/pypy/module/hpy_universal/test/support.py b/pypy/module/hpy_universal/test/support.py
--- a/pypy/module/hpy_universal/test/support.py
+++ b/pypy/module/hpy_universal/test/support.py
@@ -25,14 +25,6 @@
             pytest.skip()
         cls.compiler = ExtensionCompiler(udir)
 
-        w_FakeSpec = cls.space.appexec([], """():
-            class FakeSpec:
-                def __init__(self, name, origin):
-                    self.name = name
-                    self.origin = origin
-            return FakeSpec
-        """)
-
         @unwrap_spec(source_template='text', name='text')
         def descr_make_module(space, source_template, name='mytest'):
             source = _support.expand_template(source_template, name)
@@ -46,11 +38,10 @@
             so_filename = _support.c_compile(str(tmpdir), ext, compiler_verbose=False,
                                     universal_mode=True)
             #
-            w_mod = space.appexec(
-                [w_FakeSpec, space.newtext(so_filename), space.newtext(name)],
-                """(FakeSpec, path, modname):
-                    from hpy_universal import load_from_spec
-                    return load_from_spec(FakeSpec(modname, path))
+            w_mod = space.appexec([space.newtext(so_filename), space.newtext(name)],
+                """(path, modname):
+                    import hpy_universal
+                    return hpy_universal.load(modname, path)
                 """
             )
             return w_mod

From pypy.commits at gmail.com  Mon Nov 18 06:28:45 2019
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 18 Nov 2019 03:28:45 -0800 (PST)
Subject: [pypy-commit] pypy hpy: reduce code duplication and reuse most of
 the logic which is already in support.py
Message-ID: <5dd2806d.1c69fb81.9ae47.4c10@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98107:be9be5d98bb9
Date: 2019-11-18 11:37 +0100
http://bitbucket.org/pypy/pypy/changeset/be9be5d98bb9/

Log:	reduce code duplication and reuse most of the logic which is already
	in support.py

diff --git a/pypy/module/hpy_universal/test/support.py b/pypy/module/hpy_universal/test/support.py
--- a/pypy/module/hpy_universal/test/support.py
+++ b/pypy/module/hpy_universal/test/support.py
@@ -6,38 +6,27 @@
 from pypy.module.hpy_universal._vendored.test import support as _support
 
 
+class HPyTest(object):
 
-class ExtensionCompiler(object):
-    def __init__(self, base_dir):
-        self.base_dir = base_dir
-
-    def get_builddir(self, name='mytest'):
-        builddir = py.path.local.make_numbered_dir(
-            rootdir=py.path.local(self.base_dir),
-            prefix=name + '-',
-            keep=0)  # keep everything
-        return builddir
-
-
-class HPyTest(object):
     def setup_class(cls):
         if cls.runappdirect:
             pytest.skip()
-        cls.compiler = ExtensionCompiler(udir)
+
+    def setup_method(self, meth):
+        # we don't have fixtures in AppTests, so setup_method is a poor's man
+        # way of providing the 'make_module' fixture that HPyTest expects.  In
+        # theory it would be nice to call interp2app only once for the entire
+        # class instead of once per method, but I quickly benchmarked it and
+        # it does not seem to have a noticeable impact on the total time
+        # needed to run the tests
+        tmpdir = py.path.local.make_numbered_dir(rootdir=udir,
+                                                 prefix=meth.__name__ + '-',
+                                                 keep=0)  # keep everything
+        compiler = _support.ExtensionCompiler(tmpdir, 'universal', INCLUDE_DIR)
 
         @unwrap_spec(source_template='text', name='text')
         def descr_make_module(space, source_template, name='mytest'):
-            source = _support.expand_template(source_template, name)
-            tmpdir = cls.compiler.get_builddir()
-            filename = tmpdir.join(name+ '.c')
-            filename.write(source)
-            #
-            ext = _support.get_extension(str(filename), name, include_dirs=[INCLUDE_DIR],
-                                extra_compile_args=['-Wfatal-errors', '-g', '-Og'],
-                                extra_link_args=['-g'])
-            so_filename = _support.c_compile(str(tmpdir), ext, compiler_verbose=False,
-                                    universal_mode=True)
-            #
+            so_filename = compiler.compile_module(source_template, name)
             w_mod = space.appexec([space.newtext(so_filename), space.newtext(name)],
                 """(path, modname):
                     import hpy_universal
@@ -45,5 +34,4 @@
                 """
             )
             return w_mod
-        cls.w_make_module = cls.space.wrap(interp2app(descr_make_module))
-
+        self.w_make_module = self.space.wrap(interp2app(descr_make_module))

From pypy.commits at gmail.com  Mon Nov 18 06:28:46 2019
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 18 Nov 2019 03:28:46 -0800 (PST)
Subject: [pypy-commit] pypy hpy: remove the test duplication: we can now
 reuse directly the tests which we vendored from hpy
Message-ID: <5dd2806e.1c69fb81.9dc83.e43f@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98108:0fee85415527
Date: 2019-11-18 12:09 +0100
http://bitbucket.org/pypy/pypy/changeset/0fee85415527/

Log:	remove the test duplication: we can now reuse directly the tests
	which we vendored from hpy

diff --git a/pypy/module/hpy_universal/test/support.py b/pypy/module/hpy_universal/test/support.py
--- a/pypy/module/hpy_universal/test/support.py
+++ b/pypy/module/hpy_universal/test/support.py
@@ -6,19 +6,21 @@
 from pypy.module.hpy_universal._vendored.test import support as _support
 
 
-class HPyTest(object):
+class HPyAppTest(object):
+
+    @pytest.fixture
+    def compiler(self):
+        # see setup_method below
+        return 'The fixture "compiler" is not used on pypy'
 
     def setup_class(cls):
         if cls.runappdirect:
             pytest.skip()
 
     def setup_method(self, meth):
-        # we don't have fixtures in AppTests, so setup_method is a poor's man
-        # way of providing the 'make_module' fixture that HPyTest expects.  In
-        # theory it would be nice to call interp2app only once for the entire
-        # class instead of once per method, but I quickly benchmarked it and
-        # it does not seem to have a noticeable impact on the total time
-        # needed to run the tests
+        # it would be nice to use the 'compiler' fixture to provide
+        # make_module as the std HPyTest do. Howwever, we don't have the space
+        # yet, so it is much easier to prove make_module() here
         tmpdir = py.path.local.make_numbered_dir(rootdir=udir,
                                                  prefix=meth.__name__ + '-',
                                                  keep=0)  # keep everything
diff --git a/pypy/module/hpy_universal/test/test_basic.py b/pypy/module/hpy_universal/test/test_basic.py
--- a/pypy/module/hpy_universal/test/test_basic.py
+++ b/pypy/module/hpy_universal/test/test_basic.py
@@ -1,133 +1,6 @@
-from .support import HPyTest
+import pytest
+from pypy.module.hpy_universal._vendored.test.test_basic import TestBasic as _TestBasic
+from .support import HPyAppTest
 
-class AppTestBasic(HPyTest):
+class AppTestBasic(HPyAppTest, _TestBasic):
     spaceconfig = {'usemodules': ['hpy_universal']}
-    def test_import(self):
-        import hpy_universal
-
-    def test_empty_module(self):
-        import sys
-        mod = self.make_module("""
-            @INIT
-        """)
-        assert type(mod) is type(sys)
-
-    def test_empty_module_initialization(self):
-        skip("FIXME")
-        import sys
-        mod = self.make_module("""
-            @INIT
-        """)
-        assert type(mod) is type(sys)
-        assert mod.__loader__.name == 'mytest'
-        assert mod.__spec__.loader is mod.__loader__
-        assert mod.__file__
-
-    def test_different_name(self):
-        mod = self.make_module("""
-            @INIT
-        """, name="foo")
-        assert mod.__name__ == "foo"
-
-    def test_noop_function(self):
-        mod = self.make_module("""
-            HPy_FUNCTION(f)
-            static HPy f_impl(HPyContext ctx, HPy self, HPy args)
-            {
-                return HPyNone_Get(ctx);
-            }
-            @EXPORT f METH_NOARGS
-            @INIT
-        """)
-        assert mod.f() is None
-
-    def test_self_is_module(self):
-        mod = self.make_module("""
-            HPy_FUNCTION(f)
-            static HPy f_impl(HPyContext ctx, HPy self, HPy args)
-            {
-                return HPy_Dup(ctx, self);
-            }
-            @EXPORT f METH_NOARGS
-            @INIT
-        """)
-        assert mod.f() is mod
-
-    def test_identity_function(self):
-        mod = self.make_module("""
-            HPy_FUNCTION(f)
-            static HPy f_impl(HPyContext ctx, HPy self, HPy arg)
-            {
-                return HPy_Dup(ctx, arg);
-            }
-            @EXPORT f METH_O
-            @INIT
-        """)
-        x = object()
-        assert mod.f(x) is x
-
-    def test_long_aslong(self):
-        mod = self.make_module("""
-            HPy_FUNCTION(f)
-            static HPy f_impl(HPyContext ctx, HPy self, HPy arg)
-            {
-                long a = HPyLong_AsLong(ctx, arg);
-                return HPyLong_FromLong(ctx, a * 2);
-            }
-            @EXPORT f METH_O
-            @INIT
-        """)
-        assert mod.f(45) == 90
-
-    def test_wrong_number_of_arguments(self):
-        # XXX: this test was manually modified to turn pytest.raises into raises :(
-        mod = self.make_module("""
-            HPy_FUNCTION(f_noargs)
-            static HPy f_noargs_impl(HPyContext ctx, HPy self, HPy args)
-            {
-                return HPyNone_Get(ctx);
-            }
-            HPy_FUNCTION(f_o)
-            static HPy f_o_impl(HPyContext ctx, HPy self, HPy args)
-            {
-                return HPyNone_Get(ctx);
-            }
-            @EXPORT f_noargs METH_NOARGS
-            @EXPORT f_o METH_O
-            @INIT
-        """)
-        with raises(TypeError):
-            mod.f_noargs(1)
-        with raises(TypeError):
-            mod.f_o()
-        with raises(TypeError):
-            mod.f_o(1, 2)
-
-    def test_close(self):
-        mod = self.make_module("""
-            HPy_FUNCTION(f)
-            static HPy f_impl(HPyContext ctx, HPy self, HPy arg)
-            {
-                HPy one = HPyLong_FromLong(ctx, 1);
-                if (HPy_IsNull(one))
-                    return HPy_NULL;
-                HPy res = HPyNumber_Add(ctx, arg, one);
-                HPy_Close(ctx, one);
-                return res;
-            }
-            @EXPORT f METH_O
-            @INIT
-        """)
-        assert mod.f(41.5) == 42.5
-
-    def test_string(self):
-        mod = self.make_module("""
-            HPy_FUNCTION(f)
-            static HPy f_impl(HPyContext ctx, HPy self, HPy args)
-            {
-                return HPyUnicode_FromString(ctx, "foobar");
-            }
-            @EXPORT f METH_NOARGS
-            @INIT
-        """)
-        assert mod.f() == "foobar"

From pypy.commits at gmail.com  Mon Nov 18 06:37:36 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 18 Nov 2019 03:37:36 -0800 (PST)
Subject: [pypy-commit] pypy default: issue 3117: calling PyType_Ready on a
 subclass with a partially built tp_base
Message-ID: <5dd28280.1c69fb81.1d197.d144@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r98109:7fd34a4a2b89
Date: 2019-11-18 04:33 -0700
http://bitbucket.org/pypy/pypy/changeset/7fd34a4a2b89/

Log:	issue 3117: calling PyType_Ready on a subclass with a partially
	built tp_base

diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -912,8 +912,10 @@
         # While this is a hack, cpython does it as well.
         w_metatype = space.w_type
 
-    w_obj = space.allocate_instance(W_PyCTypeObject, w_metatype)
-    track_reference(space, py_obj, w_obj)
+    w_obj = rawrefcount.to_obj(W_PyCTypeObject, py_obj)
+    if w_obj is None:
+        w_obj = space.allocate_instance(W_PyCTypeObject, w_metatype)
+        track_reference(space, py_obj, w_obj)
     # __init__ wraps all slotdefs functions from py_type via add_operators
     w_obj.__init__(space, py_type)
     w_obj.ready()
@@ -940,7 +942,7 @@
     Sets up tp_bases, necessary before creating the interpreter type.
     """
     base = pto.c_tp_base
-    base_pyo = rffi.cast(PyObject, pto.c_tp_base)
+    base_pyo = rffi.cast(PyObject, base)
     if base and not base.c_tp_flags & Py_TPFLAGS_READY:
         type_realize(space, base_pyo)
     if base and not pto.c_ob_type: # will be filled later

From pypy.commits at gmail.com  Mon Nov 18 06:37:38 2019
From: pypy.commits at gmail.com (mattip)
Date: Mon, 18 Nov 2019 03:37:38 -0800 (PST)
Subject: [pypy-commit] pypy py3.6: merge default into py3.6
Message-ID: <5dd28282.1c69fb81.629ee.69ad@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r98110:dde73a3a316b
Date: 2019-11-18 04:36 -0700
http://bitbucket.org/pypy/pypy/changeset/dde73a3a316b/

Log:	merge default into py3.6

diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -267,7 +267,8 @@
 if sys.platform == 'win32':
     DEFAULT_SFLAGS_PLATFORM = SF_MSVC_BITFIELDS
 else:
-    if rffi_platform.getdefined('__arm__', ''):
+    if (rffi_platform.getdefined('__arm__', '') or
+        rffi_platform.getdefined('__aarch64__', '')):
         DEFAULT_SFLAGS_PLATFORM = SF_GCC_ARM_BITFIELDS
     else:
         DEFAULT_SFLAGS_PLATFORM = SF_GCC_X86_BITFIELDS
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -793,8 +793,10 @@
         # While this is a hack, cpython does it as well.
         w_metatype = space.w_type
 
-    w_obj = space.allocate_instance(W_PyCTypeObject, w_metatype)
-    track_reference(space, py_obj, w_obj)
+    w_obj = rawrefcount.to_obj(W_PyCTypeObject, py_obj)
+    if w_obj is None:
+        w_obj = space.allocate_instance(W_PyCTypeObject, w_metatype)
+        track_reference(space, py_obj, w_obj)
     # __init__ wraps all slotdefs functions from py_type via add_operators
     w_obj.__init__(space, py_type)
     w_obj.ready()
@@ -818,7 +820,7 @@
     Sets up tp_bases, necessary before creating the interpreter type.
     """
     base = pto.c_tp_base
-    base_pyo = rffi.cast(PyObject, pto.c_tp_base)
+    base_pyo = rffi.cast(PyObject, base)
     if base and not base.c_tp_flags & Py_TPFLAGS_READY:
         type_realize(space, base_pyo)
     if base and not pto.c_ob_type: # will be filled later
diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -18,6 +18,20 @@
         return not detect_simd_z()
     return True
 
+def align_check(input):
+    if platform.machine().startswith('x86'):
+        return ""
+    if sys.maxsize > 2**32:
+        mask = 7
+    else:
+        mask = 3
+    return """
+        i10096 = int_and(%s, %d)
+        i10097 = int_is_zero(i10096)
+        guard_true(i10097, descr=...)
+    """ % (input, mask)
+
+
 @py.test.mark.skipif(True, reason='no _numpypy on pypy3')
 class TestMicroNumPy(BaseTestPyPyC):
 
@@ -144,8 +158,9 @@
             i9 = getfield_gc_i(p4, descr=<FieldU pypy.module.micronumpy.concrete.BaseConcreteArray.inst_storage \d+ pure>)
             i10 = getfield_gc_i(p6, descr=<FieldU pypy.module.micronumpy.descriptor.W_Dtype.inst_byteorder \d+ pure>)
             i12 = int_eq(i10, 61)
-            i14 = int_eq(i10, %d)
+            i14 = int_eq(i10, %(bit)d)
             i15 = int_or(i12, i14)
+            %(align_check)s
             f16 = raw_load_f(i9, i5, descr=<ArrayF \d+>)
             guard_true(i15, descr=...)
             guard_not_invalidated(descr=...)
@@ -177,7 +192,7 @@
             setfield_gc(p34, i30, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_offset \d+>)
             }}}
             jump(..., descr=...)
-        """ % (bit,))
+        """ % {'align_check': align_check('i5'), 'bit': bit})
 
     def test_reduce_logical_and(self):
         def main():
@@ -190,6 +205,7 @@
         assert len(log.loops) == 1
         loop = log._filter(log.loops[0])
         loop.match("""
+            %(align_check)s
             f31 = raw_load_f(i9, i29, descr=<ArrayF 8>)
             guard_not_invalidated(descr=...)
             i32 = float_ne(f31, 0.000000)
@@ -199,7 +215,7 @@
             i38 = int_ge(i36, i30)
             guard_false(i38, descr=...)
             jump(..., descr=...)
-            """)
+            """ % {'align_check': align_check('i29')})
         # vector version
         #assert loop.match("""
         #    guard_not_invalidated(descr=...)
@@ -311,6 +327,7 @@
             guard_not_invalidated(descr=...)
             i88 = int_ge(i87, i59)
             guard_false(i88, descr=...)
+            %(align_check)s
             f90 = raw_load_f(i67, i89, descr=<ArrayF 8>)
             i91 = int_add(i87, 1)
             i93 = int_add(i89, 8)
@@ -321,7 +338,7 @@
             i96 = int_lt(i95, 0)
             guard_false(i96, descr=...)
             jump(..., descr=...)
-        """)
+        """ % {"align_check": align_check('i89')})
 
     def test_array_flatiter_getitem_single(self):
         def main():
@@ -343,6 +360,7 @@
             guard_true(i126, descr=...)
             i128 = int_mul(i117, i59)
             i129 = int_add(i55, i128)
+            %(align_check)s
             f149 = raw_load_f(i100, i129, descr=<ArrayF 8>)
             i151 = int_add(i117, 1)
             setfield_gc(p156, i55, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_offset .+>)
@@ -350,7 +368,7 @@
             setarrayitem_gc(p150, 0, 0, descr=<ArrayS .+>)
             --TICK--
             jump(..., descr=...)
-        """)
+        """ % {'align_check': align_check('i129')})
 
     def test_array_flatiter_setitem_single(self):
         def main():
@@ -373,6 +391,7 @@
             i131 = int_mul(i120, i57)
             i132 = int_add(i53, i131)
             guard_not_invalidated(descr=...)
+            %(align_check)s
             raw_store(i103, i132, 42.000000, descr=<ArrayF 8>)
             i153 = int_add(i120, 1)
             i154 = getfield_raw_i(#, descr=<FieldS pypysig_long_struct.c_value 0>)
@@ -382,7 +401,7 @@
             i157 = int_lt(i154, 0)
             guard_false(i157, descr=...)
             jump(..., descr=...)
-        """)
+        """ % {'align_check': align_check('i132')})
 
     def test_mixed_div(self):
         N = 1500
@@ -406,6 +425,7 @@
             guard_false(i94, descr=...)
             i96 = int_mul(i91, i58)
             i97 = int_add(i51, i96)
+            %(align_check)s
             f98 = raw_load_f(i63, i97, descr=<ArrayF 8>)
             guard_not_invalidated(descr=...)
             f100 = float_mul(f98, 0.500000)
@@ -423,4 +443,4 @@
             i107 = int_lt(i106, 0)
             guard_false(i107, descr=...)
             jump(..., descr=...)
-        """)
+        """ % {'align_check': align_check('i97')})
diff --git a/rpython/jit/backend/x86/rx86.py b/rpython/jit/backend/x86/rx86.py
--- a/rpython/jit/backend/x86/rx86.py
+++ b/rpython/jit/backend/x86/rx86.py
@@ -699,6 +699,9 @@
     TEST_ai = insn(rex_w, '\xF7', orbyte(0<<3), mem_reg_plus_scaled_reg_plus_const(1), immediate(2))
     TEST_mi = insn(rex_w, '\xF7', orbyte(0<<3), mem_reg_plus_const(1), immediate(2))
     TEST_ji = insn(rex_w, '\xF7', orbyte(0<<3), abs_(1), immediate(2))
+    TEST_ri = insn(rex_w, '\xF7', orbyte(0<<3), register(1), '\xC0', immediate(2))
+    TEST_bi = insn(rex_w, '\xF7', orbyte(0<<3), stack_bp(1), immediate(2))
+    TEST_br = insn(rex_w, '\x85', register(2,8), stack_bp(1))
 
     BTS_mr = insn(rex_w, '\x0F\xAB', register(2,8), mem_reg_plus_const(1))
     BTS_jr = insn(rex_w, '\x0F\xAB', register(2,8), abs_(1))
diff --git a/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py b/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py
--- a/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py
+++ b/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py
@@ -311,6 +311,8 @@
                 return []   # MOV AL, [immediate]: there is a special encoding
             if methname == 'MOV8_jr' and args[1] == rx86.R.al:
                 return []   # MOV [immediate], AL: there is a special encoding
+            if methname == 'TEST_ri' and args[0] == rx86.R.eax:
+                return []  # TEST EAX, constant: there is a special encoding
 
             return [args]
 

From pypy.commits at gmail.com  Mon Nov 18 07:09:42 2019
From: pypy.commits at gmail.com (arigo)
Date: Mon, 18 Nov 2019 04:09:42 -0800 (PST)
Subject: [pypy-commit] pypy hpy: update pyhandle/hpy 56a54e1
Message-ID: <5dd28a06.1c69fb81.e09d9.e199@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: hpy
Changeset: r98111:635f3981943c
Date: 2019-11-18 12:53 +0100
http://bitbucket.org/pypy/pypy/changeset/635f3981943c/

Log:	update pyhandle/hpy 56a54e1

diff --git a/pypy/module/hpy_universal/_vendored/test/conftest.py b/pypy/module/hpy_universal/_vendored/test/conftest.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/_vendored/test/conftest.py
@@ -0,0 +1,26 @@
+import pytest
+from .support import ExtensionCompiler
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--correct", action="store_true",
+        help="Test against headers installed through hpy_devel"
+    )
+
+ at pytest.fixture(scope='session')
+def hpy_include_dir(request):
+    if request.config.getoption('--correct'):
+        from hpy_devel import get_include
+        return get_include()
+    else:
+        import os
+        THIS_DIR = os.path.dirname(__file__)
+        return os.path.join(THIS_DIR, '../hpy-api/hpy_devel/include')
+
+ at pytest.fixture(params=['cpython', 'universal'])
+def abimode(request):
+    return request.param
+
+ at pytest.fixture
+def compiler(tmpdir, abimode, hpy_include_dir):
+    return ExtensionCompiler(tmpdir, abimode, hpy_include_dir)
diff --git a/pypy/module/hpy_universal/_vendored/test/support.py b/pypy/module/hpy_universal/_vendored/test/support.py
--- a/pypy/module/hpy_universal/_vendored/test/support.py
+++ b/pypy/module/hpy_universal/_vendored/test/support.py
@@ -78,7 +78,8 @@
         #
         ext = get_extension(str(filename), name,
                             include_dirs=[self.include_dir],
-                            extra_compile_args=['-Wfatal-errors'])
+                            extra_compile_args=['-Wfatal-errors', '-g', '-Og'],
+                            extra_link_args=['-g'])
         so_filename = c_compile(str(self.tmpdir), ext, compiler_verbose=False,
                                 universal_mode=self.universal_mode)
         return so_filename
@@ -196,13 +197,15 @@
     objects = compiler.compile(ext.sources,
                                output_dir=tmpdir,
                                macros=[('HPY_UNIVERSAL_ABI', None)],
-                               include_dirs=include_dirs)
+                               include_dirs=include_dirs,
+                               extra_preargs=ext.extra_compile_args)
 
     filename = ext.name + '.hpy.so'
     compiler.link(compiler.SHARED_LIBRARY,
                   objects,
                   filename,
-                  tmpdir
+                  tmpdir,
+                  extra_preargs=ext.extra_link_args,
                   # export_symbols=...
                   )
     return os.path.join(tmpdir, filename)
diff --git a/pypy/module/hpy_universal/_vendored/test/test_basic.py b/pypy/module/hpy_universal/_vendored/test/test_basic.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/_vendored/test/test_basic.py
@@ -0,0 +1,160 @@
+"""
+NOTE: this tests are also meant to be run as PyPy "applevel" tests.
+
+This means that global imports will NOT be visible inside the test
+functions. In particular, you have to "import pytest" inside the test in order
+to be able to use e.g. pytest.raises (which on PyPy will be implemented by a
+"fake pytest module")
+"""
+from .support import HPyTest
+
+
+class TestBasic(HPyTest):
+
+    def test_empty_module(self):
+        import sys
+        mod = self.make_module("""
+            @INIT
+        """)
+        assert type(mod) is type(sys)
+
+    def test_different_name(self):
+        mod = self.make_module("""
+            @INIT
+        """, name="foo")
+        assert mod.__name__ == "foo"
+
+    def test_noop_function(self):
+        mod = self.make_module("""
+            HPy_METH_NOARGS(f)
+            static HPy f_impl(HPyContext ctx, HPy self)
+            {
+                return HPy_Dup(ctx, ctx->h_None);
+            }
+            @EXPORT f METH_NOARGS
+            @INIT
+        """)
+        assert mod.f() is None
+
+    def test_self_is_module(self):
+        mod = self.make_module("""
+            HPy_METH_NOARGS(f)
+            static HPy f_impl(HPyContext ctx, HPy self)
+            {
+                return HPy_Dup(ctx, self);
+            }
+            @EXPORT f METH_NOARGS
+            @INIT
+        """)
+        assert mod.f() is mod
+
+    def test_identity_function(self):
+        mod = self.make_module("""
+            HPy_METH_O(f)
+            static HPy f_impl(HPyContext ctx, HPy self, HPy arg)
+            {
+                return HPy_Dup(ctx, arg);
+            }
+            @EXPORT f METH_O
+            @INIT
+        """)
+        x = object()
+        assert mod.f(x) is x
+
+    def test_long_aslong(self):
+        mod = self.make_module("""
+            HPy_METH_O(f)
+            static HPy f_impl(HPyContext ctx, HPy self, HPy arg)
+            {
+                long a = HPyLong_AsLong(ctx, arg);
+                return HPyLong_FromLong(ctx, a * 2);
+            }
+            @EXPORT f METH_O
+            @INIT
+        """)
+        assert mod.f(45) == 90
+
+    def test_wrong_number_of_arguments(self):
+        import pytest
+        mod = self.make_module("""
+            HPy_METH_NOARGS(f_noargs)
+            static HPy f_noargs_impl(HPyContext ctx, HPy self)
+            {
+                return HPy_Dup(ctx, ctx->h_None);
+            }
+            HPy_METH_O(f_o)
+            static HPy f_o_impl(HPyContext ctx, HPy self, HPy arg)
+            {
+                return HPy_Dup(ctx, ctx->h_None);
+            }
+            @EXPORT f_noargs METH_NOARGS
+            @EXPORT f_o METH_O
+            @INIT
+        """)
+        with pytest.raises(TypeError):
+            mod.f_noargs(1)
+        with pytest.raises(TypeError):
+            mod.f_o()
+        with pytest.raises(TypeError):
+            mod.f_o(1, 2)
+
+    def test_many_int_arguments(self):
+        mod = self.make_module("""
+            HPy_METH_VARARGS(f)
+            static HPy f_impl(HPyContext ctx, HPy self,
+                              HPy *args, HPy_ssize_t nargs)
+            {
+                long a, b, c, d, e;
+                if (!HPyArg_Parse(ctx, args, nargs, "lllll",
+                                  &a, &b, &c, &d, &e))
+                    return HPy_NULL;
+                return HPyLong_FromLong(ctx,
+                    10000*a + 1000*b + 100*c + 10*d + e);
+            }
+            @EXPORT f METH_VARARGS
+            @INIT
+        """)
+        assert mod.f(4, 5, 6, 7, 8) == 45678
+
+    def test_close(self):
+        mod = self.make_module("""
+            HPy_METH_O(f)
+            static HPy f_impl(HPyContext ctx, HPy self, HPy arg)
+            {
+                HPy one = HPyLong_FromLong(ctx, 1);
+                if (HPy_IsNull(one))
+                    return HPy_NULL;
+                HPy res = HPyNumber_Add(ctx, arg, one);
+                HPy_Close(ctx, one);
+                return res;
+            }
+            @EXPORT f METH_O
+            @INIT
+        """)
+        assert mod.f(41.5) == 42.5
+
+    def test_string(self):
+        mod = self.make_module("""
+            HPy_METH_NOARGS(f)
+            static HPy f_impl(HPyContext ctx, HPy self)
+            {
+                return HPyUnicode_FromString(ctx, "foobar");
+            }
+            @EXPORT f METH_NOARGS
+            @INIT
+        """)
+        assert mod.f() == "foobar"
+
+    def test_bool(self):
+        mod = self.make_module("""
+            HPy_METH_O(f)
+            static HPy f_impl(HPyContext ctx, HPy self, HPy arg)
+            {
+                int cond = HPyLong_AsLong(ctx, arg) > 5;
+                return HPy_Dup(ctx, cond ? ctx->h_True : ctx->h_False);
+            }
+            @EXPORT f METH_O
+            @INIT
+        """)
+        assert mod.f(4) is False
+        assert mod.f(6) is True
diff --git a/pypy/module/hpy_universal/_vendored/test/test_cpy_compat.py b/pypy/module/hpy_universal/_vendored/test/test_cpy_compat.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/_vendored/test/test_cpy_compat.py
@@ -0,0 +1,105 @@
+from .support import HPyTest
+
+
+class TestCPythonCompatibility(HPyTest):
+
+    def test_frompyobject(self):
+        mod = self.make_module("""
+            #include <Python.h>
+            HPy_METH_NOARGS(f)
+            static HPy f_impl(HPyContext ctx, HPy self)
+            {
+                PyObject *o = PyList_New(0);
+
+                Py_ssize_t initial_refcount = o->ob_refcnt;
+                HPy h = HPy_FromPyObject(ctx, o);
+                Py_ssize_t final_refcount = o->ob_refcnt;
+
+                PyList_Append(o, PyLong_FromSsize_t(final_refcount -
+                                                    initial_refcount));
+                Py_DECREF(o);
+                return h;
+            }
+            @EXPORT f METH_NOARGS
+            @INIT
+        """)
+        x = mod.f()
+        assert x == [+1]
+
+    def test_hpy_close(self):
+        mod = self.make_module("""
+            #include <Python.h>
+            HPy_METH_NOARGS(f)
+            static HPy f_impl(HPyContext ctx, HPy self)
+            {
+                PyObject *o = PyList_New(0);
+
+                HPy h = HPy_FromPyObject(ctx, o);
+                Py_ssize_t initial_refcount = o->ob_refcnt;
+                HPy_Close(ctx, h);
+                Py_ssize_t final_refcount = o->ob_refcnt;
+
+                Py_DECREF(o);
+                return HPyLong_FromLong(ctx, (long)(final_refcount -
+                                                    initial_refcount));
+            }
+            @EXPORT f METH_NOARGS
+            @INIT
+        """)
+        assert mod.f() == -1
+
+    def test_hpy_dup(self):
+        mod = self.make_module("""
+            #include <Python.h>
+            HPy_METH_NOARGS(f)
+            static HPy f_impl(HPyContext ctx, HPy self)
+            {
+                PyObject *o = PyList_New(0);
+
+                HPy h = HPy_FromPyObject(ctx, o);
+                Py_ssize_t initial_refcount = o->ob_refcnt;
+                HPy h2 = HPy_Dup(ctx, h);
+                Py_ssize_t final_refcount = o->ob_refcnt;
+
+                HPy_Close(ctx, h);
+                HPy_Close(ctx, h2);
+                Py_DECREF(o);
+                return HPyLong_FromLong(ctx, (long)(final_refcount -
+                                                    initial_refcount));
+            }
+            @EXPORT f METH_NOARGS
+            @INIT
+        """)
+        assert mod.f() == +1
+
+    def test_many_handles(self):
+        mod = self.make_module("""
+            #include <Python.h>
+            #define NUM_HANDLES  10000
+
+            HPy_METH_NOARGS(f)
+            static HPy f_impl(HPyContext ctx, HPy self)
+            {
+                PyObject *o = PyList_New(0);
+
+                Py_ssize_t result = -42;
+                HPy handles[NUM_HANDLES];
+                int i;
+                Py_ssize_t initial_refcount = o->ob_refcnt;
+                for (i = 0; i < NUM_HANDLES; i++)
+                    handles[i] = HPy_FromPyObject(ctx, o);
+                for (i = 0; i < NUM_HANDLES; i++)
+                    if (HPy_IsNull(handles[i]))
+                        goto error;
+                for (i = 0; i < NUM_HANDLES; i++)
+                    HPy_Close(ctx, handles[i]);
+                Py_ssize_t final_refcount = o->ob_refcnt;
+                result = final_refcount - initial_refcount;
+
+             error:
+                return HPyLong_FromLong(ctx, (long)result);
+            }
+            @EXPORT f METH_NOARGS
+            @INIT
+        """)
+        assert mod.f() == 0
diff --git a/pypy/module/hpy_universal/_vendored/test/test_importing.py b/pypy/module/hpy_universal/_vendored/test/test_importing.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/_vendored/test/test_importing.py
@@ -0,0 +1,30 @@
+import pytest
+from .support import HPyTest
+
+# this function should probably goes somewhere into hpy_universal and/or and
+# hpy package and/or an import hook, or whatever. I do not want to think about
+# this now.
+def import_module_properly(mod):
+    raise NotImplementedError("fix me eventually")
+
+# this was moved from support.py, where it did not belong
+## class HPyLoader(ExtensionFileLoader):
+##     def create_module(self, spec):
+##         import hpy_universal
+##         return hpy_universal.load_from_spec(spec)
+
+
+class TestImporting(HPyTest):
+
+    @pytest.mark.xfail
+    def test_importing_attributes(self):
+        import sys
+        modname = 'mytest'
+        so_filename = self.compile_module("""
+            @INIT
+        """, name=modname)
+        mod = import_module_properly(so_filename, modname)
+        assert mod in sys.modules
+        assert mod.__loader__.name == 'mytest'
+        assert mod.__spec__.loader is mod.__loader__
+        assert mod.__file__
diff --git a/pypy/module/hpy_universal/_vendored/test/test_support.py b/pypy/module/hpy_universal/_vendored/test/test_support.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/_vendored/test/test_support.py
@@ -0,0 +1,16 @@
+from . import support
+
+
+def test_expand_template():
+    expanded = support.expand_template("""
+        @EXPORT test_f METH_O
+        some more C stuff
+        @INIT
+    """, name='mytest')
+    methods = '{"test_f", test_f, METH_O, NULL},'
+    init_code = support.INIT_TEMPLATE % {'methods': methods, 'name': 'mytest'}
+    assert expanded.rstrip() == f"""#include <hpy.h>
+
+        some more C stuff
+{init_code}
+""".rstrip()

From pypy.commits at gmail.com  Mon Nov 18 07:09:44 2019
From: pypy.commits at gmail.com (arigo)
Date: Mon, 18 Nov 2019 04:09:44 -0800 (PST)
Subject: [pypy-commit] pypy hpy: Bah,
 fix the update_vendored script and really update everything to
 56a54e1
Message-ID: <5dd28a08.1c69fb81.90508.1fc9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: hpy
Changeset: r98112:0e8678278177
Date: 2019-11-18 13:03 +0100
http://bitbucket.org/pypy/pypy/changeset/0e8678278177/

Log:	Bah, fix the update_vendored script and really update everything to
	56a54e1

diff --git a/pypy/module/hpy_universal/_vendored/include/cpython/hpy.h b/pypy/module/hpy_universal/_vendored/include/cpython/hpy.h
--- a/pypy/module/hpy_universal/_vendored/include/cpython/hpy.h
+++ b/pypy/module/hpy_universal/_vendored/include/cpython/hpy.h
@@ -6,7 +6,7 @@
 
 /* XXX: it would be nice if we could include hpy.h WITHOUT bringing in all the
    stuff from Python.h, to make sure that people don't use the CPython API by
-   mistake. How to achieve it, though? */
+   mistake. How to achieve it, though? Is defining Py_LIMITED_API enough? */
 
 /* XXX: should we:
  *    - enforce PY_SSIZE_T_CLEAN in hpy
@@ -24,12 +24,7 @@
 
 
 typedef struct { PyObject *_o; } HPy;
-typedef long HPyContext;
-
-HPyAPI_FUNC(HPyContext)
-_HPyGetContext(void) {
-    return 42;
-}
+typedef Py_ssize_t HPy_ssize_t;
 
 /* For internal usage only. These should be #undef at the end of this header.
    If you need to convert HPy to PyObject* and vice-versa, you should use the
@@ -38,6 +33,24 @@
 #define _h2py(x) (x._o)
 #define _py2h(o) ((HPy){o})
 
+typedef struct _HPyContext_s {
+    HPy h_None;
+    HPy h_True;
+    HPy h_False;
+} *HPyContext;
+
+/* XXX! should be defined only once, not once for every .c! */
+static struct _HPyContext_s _global_ctx = {
+    .h_None = _py2h(Py_None),
+    .h_True = _py2h(Py_True),
+    .h_False = _py2h(Py_False),
+};
+
+HPyAPI_FUNC(HPyContext)
+_HPyGetContext(void) {
+    return &_global_ctx;
+}
+
 
 #define HPy_NULL ((HPy){NULL})
 #define HPy_IsNull(x) ((x)._o == NULL)
@@ -85,23 +98,63 @@
 
 /* function declaration */
 
-#define HPy_FUNCTION(NAME)                                              \
+#define HPy_METH_NOARGS(NAME)                                           \
+    static HPy NAME##_impl(HPyContext, HPy);                            \
+    static PyObject* NAME(PyObject *self, PyObject *noargs)             \
+    {                                                                   \
+        return _h2py(NAME##_impl(_HPyGetContext(), _py2h(self)));       \
+    }
+
+#define HPy_METH_O(NAME)                                                \
     static HPy NAME##_impl(HPyContext, HPy, HPy);                       \
+    static PyObject* NAME(PyObject *self, PyObject *arg)                \
+    {                                                                   \
+        return _h2py(NAME##_impl(_HPyGetContext(), _py2h(self), _py2h(arg)));\
+    }
+
+#define HPy_METH_VARARGS(NAME)                                          \
+    static HPy NAME##_impl(HPyContext, HPy, HPy *, Py_ssize_t);         \
     static PyObject* NAME(PyObject *self, PyObject *args)               \
     {                                                                   \
-        return _h2py(NAME##_impl(_HPyGetContext(), _py2h(self), _py2h(args)));\
+        /* get the tuple elements as an array of "PyObject *", which */ \
+        /* is equivalent to an array of "HPy" with enough casting... */ \
+        HPy *items = (HPy *)&PyTuple_GET_ITEM(args, 0);                 \
+        Py_ssize_t nargs = PyTuple_GET_SIZE(args);                      \
+        return _h2py(NAME##_impl(_HPyGetContext(), _py2h(self), items, nargs));\
     }
 
 
 HPyAPI_FUNC(int)
-HPyArg_ParseTuple(HPyContext ctx, HPy args, const char *fmt, ...)
+HPyArg_Parse(HPyContext ctx, HPy *args, Py_ssize_t nargs, const char *fmt, ...)
 {
     va_list vl;
     va_start(vl, fmt);
-    int res = PyArg_VaParse(_h2py(args), fmt, vl);
+    const char *fmt1 = fmt;
+    Py_ssize_t i = 0;
+
+    while (*fmt1 != 0) {
+        if (i >= nargs) {
+            abort(); // XXX
+        }
+        switch (*fmt1++) {
+        case 'l': {
+            long *output = va_arg(vl, long *);
+            long value = PyLong_AsLong(_h2py(args[i]));
+            // XXX check for exceptions
+            *output = value;
+            break;
+        }
+        default:
+            abort();  // XXX
+        }
+        i++;
+    }
+    if (i != nargs) {
+        abort();   // XXX
+    }
+
     va_end(vl);
-    /* XXX incref all returned 'PyObject*' */
-    return res;
+    return 1;
 }
 
 
diff --git a/pypy/module/hpy_universal/_vendored/include/universal/autogen_ctx.h b/pypy/module/hpy_universal/_vendored/include/universal/autogen_ctx.h
--- a/pypy/module/hpy_universal/_vendored/include/universal/autogen_ctx.h
+++ b/pypy/module/hpy_universal/_vendored/include/universal/autogen_ctx.h
@@ -10,16 +10,18 @@
 
 struct _HPyContext_s {
     int ctx_version;
+    HPy h_None;
+    HPy h_True;
+    HPy h_False;
     HPy (*ctx_Module_Create)(HPyContext ctx, HPyModuleDef *def);
-    HPy (*ctx_None_Get)(HPyContext ctx);
     HPy (*ctx_Dup)(HPyContext ctx, HPy h);
     void (*ctx_Close)(HPyContext ctx, HPy h);
     HPy (*ctx_Long_FromLong)(HPyContext ctx, long value);
     long (*ctx_Long_AsLong)(HPyContext ctx, HPy h);
-    int (*ctx_Arg_ParseTuple)(HPyContext ctx, HPy args, const char *fmt, va_list _vl);
+    int (*ctx_Arg_Parse)(HPyContext ctx, HPy *args, HPy_ssize_t nargs, const char *fmt, va_list _vl);
     HPy (*ctx_Number_Add)(HPyContext ctx, HPy x, HPy y);
     HPy (*ctx_Unicode_FromString)(HPyContext ctx, const char *utf8);
     HPy (*ctx_FromPyObject)(HPyContext ctx, struct _object *obj);
     struct _object *(*ctx_AsPyObject)(HPyContext ctx, HPy h);
-    struct _object *(*ctx_CallRealFunctionFromTrampoline)(HPyContext ctx, struct _object *self, struct _object *args, HPyCFunction func);
+    struct _object *(*ctx_CallRealFunctionFromTrampoline)(HPyContext ctx, struct _object *self, struct _object *args, void *func, int ml_flags);
 };
diff --git a/pypy/module/hpy_universal/_vendored/include/universal/autogen_func.h b/pypy/module/hpy_universal/_vendored/include/universal/autogen_func.h
--- a/pypy/module/hpy_universal/_vendored/include/universal/autogen_func.h
+++ b/pypy/module/hpy_universal/_vendored/include/universal/autogen_func.h
@@ -12,10 +12,6 @@
      return ctx->ctx_Module_Create ( ctx, def ); 
 }
 
-static inline HPy HPyNone_Get(HPyContext ctx) {
-     return ctx->ctx_None_Get ( ctx ); 
-}
-
 static inline HPy HPy_Dup(HPyContext ctx, HPy h) {
      return ctx->ctx_Dup ( ctx, h ); 
 }
@@ -32,8 +28,8 @@
      return ctx->ctx_Long_AsLong ( ctx, h ); 
 }
 
-static inline int HPyArg_ParseTuple(HPyContext ctx, HPy args, const char *fmt, ...) {
-     va_list _vl; va_start(_vl, fmt); int _res = ctx->ctx_Arg_ParseTuple ( ctx, args, fmt, _vl ); va_end(_vl); return _res; 
+static inline int HPyArg_Parse(HPyContext ctx, HPy *args, HPy_ssize_t nargs, const char *fmt, ...) {
+     va_list _vl; va_start(_vl, fmt); int _res = ctx->ctx_Arg_Parse ( ctx, args, nargs, fmt, _vl ); va_end(_vl); return _res; 
 }
 
 static inline HPy HPyNumber_Add(HPyContext ctx, HPy x, HPy y) {
@@ -52,7 +48,7 @@
      return ctx->ctx_AsPyObject ( ctx, h ); 
 }
 
-static inline struct _object *_HPy_CallRealFunctionFromTrampoline(HPyContext ctx, struct _object *self, struct _object *args, HPyCFunction func) {
-     return ctx->ctx_CallRealFunctionFromTrampoline ( ctx, self, args, func ); 
+static inline struct _object *_HPy_CallRealFunctionFromTrampoline(HPyContext ctx, struct _object *self, struct _object *args, void *func, int ml_flags) {
+     return ctx->ctx_CallRealFunctionFromTrampoline ( ctx, self, args, func, ml_flags ); 
 }
 
diff --git a/pypy/module/hpy_universal/_vendored/include/universal/hpy.h b/pypy/module/hpy_universal/_vendored/include/universal/hpy.h
--- a/pypy/module/hpy_universal/_vendored/include/universal/hpy.h
+++ b/pypy/module/hpy_universal/_vendored/include/universal/hpy.h
@@ -9,7 +9,6 @@
 typedef struct { HPy_ssize_t _i; } HPy;
 
 typedef struct _HPyContext_s *HPyContext;
-typedef HPy (*HPyCFunction)(HPyContext, HPy self, HPy args);
 struct _object;  /* that's PyObject inside CPython */
 typedef struct _object *(*_HPy_CPyCFunction)(struct _object *self,
                                              struct _object *args);
@@ -17,7 +16,7 @@
 #define HPy_NULL ((HPy){0})
 #define HPy_IsNull(x) ((x)._i == 0)
 
-typedef void (*_HPyMethodPairFunc)(HPyCFunction *out_func,
+typedef void (*_HPyMethodPairFunc)(void **out_func,
                                    _HPy_CPyCFunction *out_trampoline);
 
 typedef struct {
@@ -55,16 +54,46 @@
 extern HPyContext _ctx_for_trampolines;
 
 
-#define HPy_FUNCTION(fnname)                                                   \
-    static HPy fnname##_impl(HPyContext ctx, HPy self, HPy args);              \
+#define HPy_METH_NOARGS(fnname)                                                \
+    static HPy fnname##_impl(HPyContext ctx, HPy self);                        \
+    static struct _object *                                                    \
+    fnname##_trampoline(struct _object *self, struct _object *noargs)          \
+    {                                                                          \
+        return _HPy_CallRealFunctionFromTrampoline(                            \
+            _ctx_for_trampolines, self, NULL, fnname##_impl, METH_NOARGS);     \
+    }                                                                          \
+    static void                                                                \
+    fnname(void **out_func, _HPy_CPyCFunction *out_trampoline)                 \
+    {                                                                          \
+        *out_func = fnname##_impl;                                             \
+        *out_trampoline = fnname##_trampoline;                                 \
+    }
+
+#define HPy_METH_O(fnname)                                                     \
+    static HPy fnname##_impl(HPyContext ctx, HPy self, HPy arg);               \
+    static struct _object *                                                    \
+    fnname##_trampoline(struct _object *self, struct _object *arg)             \
+    {                                                                          \
+        return _HPy_CallRealFunctionFromTrampoline(                            \
+            _ctx_for_trampolines, self, arg, fnname##_impl, METH_O);           \
+    }                                                                          \
+    static void                                                                \
+    fnname(void **out_func, _HPy_CPyCFunction *out_trampoline)                 \
+    {                                                                          \
+        *out_func = fnname##_impl;                                             \
+        *out_trampoline = fnname##_trampoline;                                 \
+    }
+
+#define HPy_METH_VARARGS(fnname)                                               \
+    static HPy fnname##_impl(HPyContext ctx, HPy self, HPy *args, HPy_ssize_t);\
     static struct _object *                                                    \
     fnname##_trampoline(struct _object *self, struct _object *args)            \
     {                                                                          \
         return _HPy_CallRealFunctionFromTrampoline(                            \
-            _ctx_for_trampolines, self, args, fnname##_impl);                  \
+            _ctx_for_trampolines, self, args, fnname##_impl, METH_VARARGS);    \
     }                                                                          \
     static void                                                                \
-    fnname(HPyCFunction *out_func, _HPy_CPyCFunction *out_trampoline)          \
+    fnname(void **out_func, _HPy_CPyCFunction *out_trampoline)                 \
     {                                                                          \
         *out_func = fnname##_impl;                                             \
         *out_trampoline = fnname##_trampoline;                                 \
diff --git a/pypy/module/hpy_universal/update_vendored.sh b/pypy/module/hpy_universal/update_vendored.sh
--- a/pypy/module/hpy_universal/update_vendored.sh
+++ b/pypy/module/hpy_universal/update_vendored.sh
@@ -15,7 +15,7 @@
 git -C "$HPY" --no-pager log --oneline -n 1
 git -C "$HPY" --no-pager diff --stat
 
-#cp -R "$HPY"/hpy-api/hpy_devel/include/* "$VENDORED/include"
+cp -R "$HPY"/hpy-api/hpy_devel/include/* "$VENDORED/include"
 cp -R "$HPY"/test/* "$VENDORED/test"
 
 echo

From pypy.commits at gmail.com  Mon Nov 18 07:09:46 2019
From: pypy.commits at gmail.com (arigo)
Date: Mon, 18 Nov 2019 04:09:46 -0800 (PST)
Subject: [pypy-commit] pypy hpy: Use h_None, h_False, h_True
Message-ID: <5dd28a0a.1c69fb81.a5a5b.1bc4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: hpy
Changeset: r98113:38f49b4e8f05
Date: 2019-11-18 13:07 +0100
http://bitbucket.org/pypy/pypy/changeset/38f49b4e8f05/

Log:	Use h_None, h_False, h_True

diff --git a/pypy/module/hpy_universal/handles.py b/pypy/module/hpy_universal/handles.py
--- a/pypy/module/hpy_universal/handles.py
+++ b/pypy/module/hpy_universal/handles.py
@@ -1,9 +1,16 @@
+
+CONSTANTS = [
+    ('NULL', lambda space: None),
+    ('None', lambda space: space.w_None),
+    ('False', lambda space: space.w_False),
+    ('True', lambda space: space.w_True),
+    ]
 
 
 class HandleManager:
 
     def __init__(self, space):
-        self.handles_w = [None]
+        self.handles_w = [build_value(space) for name, build_value in CONSTANTS]
         self.free_list = []
 
     def new(self, w_object):
diff --git a/pypy/module/hpy_universal/interp_hpy.py b/pypy/module/hpy_universal/interp_hpy.py
--- a/pypy/module/hpy_universal/interp_hpy.py
+++ b/pypy/module/hpy_universal/interp_hpy.py
@@ -48,10 +48,6 @@
     return handles.new(space, w_mod)
 
 
- at apifunc([llapi.HPyContext], llapi.HPy, error=0)
-def HPyNone_Get(space, ctx):
-    return handles.new(space, space.w_None)
-
 @apifunc([llapi.HPyContext, llapi.HPy], llapi.HPy, error=0)
 def HPy_Dup(space, ctx, h):
     return handles.dup(space, h)
diff --git a/pypy/module/hpy_universal/llapi.py b/pypy/module/hpy_universal/llapi.py
--- a/pypy/module/hpy_universal/llapi.py
+++ b/pypy/module/hpy_universal/llapi.py
@@ -23,13 +23,15 @@
 HPy = lltype.Signed
 HPyContextS = rffi.CStruct('_HPyContext_s',
     ('ctx_version', rffi.INT_real),
+    ('h_None', HPy),
+    ('h_True', HPy),
+    ('h_False', HPy),
     ('ctx_Module_Create', rffi.VOIDP),
-    ('ctx_None_Get', rffi.VOIDP),
     ('ctx_Dup', rffi.VOIDP),
     ('ctx_Close', rffi.VOIDP),
     ('ctx_Long_FromLong', rffi.VOIDP),
     ('ctx_Long_AsLong', rffi.VOIDP),
-    ('ctx_Arg_ParseTuple', rffi.VOIDP),
+    ('ctx_Arg_Parse', rffi.VOIDP),
     ('ctx_Number_Add', rffi.VOIDP),
     ('ctx_Unicode_FromString', rffi.VOIDP),
     ('ctx_FromPyObject', rffi.VOIDP),
diff --git a/pypy/module/hpy_universal/state.py b/pypy/module/hpy_universal/state.py
--- a/pypy/module/hpy_universal/state.py
+++ b/pypy/module/hpy_universal/state.py
@@ -4,9 +4,10 @@
 from rpython.rlib import jit
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib.objectmodel import specialize
-from pypy.module.hpy_universal import llapi
+from pypy.module.hpy_universal import llapi, handles
 
 CONTEXT_NAMES = unrolling_iterable(llapi.HPyContext.TO._names)
+CONSTANT_NAMES = unrolling_iterable([name for name, _ in handles.CONSTANTS])
 DUMMY_FUNC = lltype.FuncType([], lltype.Void)
 
 @specialize.memo()
@@ -29,24 +30,28 @@
         if self.ctx:
             return
 
+        space = self.space
         self.ctx = llapi._HPy_GetGlobalCtx()
 
         for name in CONTEXT_NAMES:
-            if name != 'c_ctx_version':
+            if name == 'c_ctx_version':
+                continue
+            if name.startswith('c_ctx_'):
                 missing_function = make_missing_function(name)
                 funcptr = llhelper(lltype.Ptr(DUMMY_FUNC), missing_function)
                 setattr(self.ctx, name, rffi.cast(rffi.VOIDP, funcptr))
+        i = 0
+        for name in CONSTANT_NAMES:
+            if name != 'NULL':
+                setattr(self.ctx, 'c_h_' + name, i)
+            i = i + 1
 
         # XXX collect all these functions automatically
         from pypy.module.hpy_universal import interp_hpy
-        space = self.space
         
         funcptr = interp_hpy.HPyModule_Create.get_llhelper(space)
         self.ctx.c_ctx_Module_Create = rffi.cast(rffi.VOIDP, funcptr)
         #
-        funcptr = interp_hpy.HPyNone_Get.get_llhelper(space)
-        self.ctx.c_ctx_None_Get = rffi.cast(rffi.VOIDP, funcptr)
-        #
         funcptr = interp_hpy.HPy_Dup.get_llhelper(space)
         self.ctx.c_ctx_Dup = rffi.cast(rffi.VOIDP, funcptr)
         #

From pypy.commits at gmail.com  Mon Nov 18 11:08:29 2019
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 18 Nov 2019 08:08:29 -0800 (PST)
Subject: [pypy-commit] pypy hpy: (antocuni,
 arigo): add a passing test which we needed to convince ourselves
 that you can raise exceptions from within llhelper()ed functions
Message-ID: <5dd2c1fd.1c69fb81.2a092.6364@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98114:3a384fd65d2e
Date: 2019-11-18 15:27 +0100
http://bitbucket.org/pypy/pypy/changeset/3a384fd65d2e/

Log:	(antocuni, arigo): add a passing test which we needed to convince
	ourselves that you can raise exceptions from within llhelper()ed
	functions

diff --git a/rpython/translator/test/test_exceptiontransform.py b/rpython/translator/test/test_exceptiontransform.py
--- a/rpython/translator/test/test_exceptiontransform.py
+++ b/rpython/translator/test/test_exceptiontransform.py
@@ -266,3 +266,26 @@
         f = self.compile(foo, [])
         res = f()
         assert res == 42
+
+    def test_llhelper_can_raise(self):
+        from rpython.rtyper.lltypesystem import lltype
+        from rpython.rtyper.annlowlevel import llhelper
+
+        def fn(a, b):
+            if b == 0:
+                raise ZeroDivisionError
+            return a/b
+
+        FN = lltype.Ptr(lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed))
+        def h(a, b):
+            fnptr = llhelper(FN, fn)
+            try:
+                return fnptr(a, b)
+            except ZeroDivisionError:
+                return -a
+
+        compiled_h = self.compile(h, [int, int])
+        res = compiled_h(39, 3)
+        assert res == 13
+        res = compiled_h(39, 0)
+        assert res == -39

From pypy.commits at gmail.com  Mon Nov 18 12:08:53 2019
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 18 Nov 2019 09:08:53 -0800 (PST)
Subject: [pypy-commit] pypy hpy: update to pyhandle/hpy c8ddac3
Message-ID: <5dd2d025.1c69fb81.abd6f.0bb2@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98115:84c205b54510
Date: 2019-11-18 17:40 +0100
http://bitbucket.org/pypy/pypy/changeset/84c205b54510/

Log:	update to pyhandle/hpy c8ddac3

diff --git a/pypy/module/hpy_universal/_vendored/include/cpython/hpy.h b/pypy/module/hpy_universal/_vendored/include/cpython/hpy.h
--- a/pypy/module/hpy_universal/_vendored/include/cpython/hpy.h
+++ b/pypy/module/hpy_universal/_vendored/include/cpython/hpy.h
@@ -37,23 +37,30 @@
     HPy h_None;
     HPy h_True;
     HPy h_False;
+    HPy h_ValueError;
 } *HPyContext;
 
 /* XXX! should be defined only once, not once for every .c! */
-static struct _HPyContext_s _global_ctx = {
-    .h_None = _py2h(Py_None),
-    .h_True = _py2h(Py_True),
-    .h_False = _py2h(Py_False),
-};
+static struct _HPyContext_s _global_ctx;
+
+#define HPy_NULL ((HPy){NULL})
+#define HPy_IsNull(x) ((x)._o == NULL)
 
 HPyAPI_FUNC(HPyContext)
 _HPyGetContext(void) {
-    return &_global_ctx;
+    HPyContext ctx = &_global_ctx;
+    if (HPy_IsNull(ctx->h_None)) {
+        // XXX: we need to find a better way to check whether the ctx is
+        // initialized or not
+        ctx->h_None = _py2h(Py_None);
+        ctx->h_True = _py2h(Py_True);
+        ctx->h_False = _py2h(Py_False);
+        ctx->h_ValueError = _py2h(PyExc_ValueError);
+    }
+    return ctx;
 }
 
 
-#define HPy_NULL ((HPy){NULL})
-#define HPy_IsNull(x) ((x)._o == NULL)
 
 HPyAPI_FUNC(HPy)
 HPyNone_Get(HPyContext ctx)
@@ -197,5 +204,11 @@
     return result;
 }
 
+HPyAPI_FUNC(void)
+HPyErr_SetString(HPyContext ctx, HPy type, const char *message)
+{
+    PyErr_SetString(_h2py(type), message);
+}
+
 
 #endif /* !HPy_CPYTHON_H */
diff --git a/pypy/module/hpy_universal/_vendored/include/universal/autogen_ctx.h b/pypy/module/hpy_universal/_vendored/include/universal/autogen_ctx.h
--- a/pypy/module/hpy_universal/_vendored/include/universal/autogen_ctx.h
+++ b/pypy/module/hpy_universal/_vendored/include/universal/autogen_ctx.h
@@ -13,6 +13,7 @@
     HPy h_None;
     HPy h_True;
     HPy h_False;
+    HPy h_ValueError;
     HPy (*ctx_Module_Create)(HPyContext ctx, HPyModuleDef *def);
     HPy (*ctx_Dup)(HPyContext ctx, HPy h);
     void (*ctx_Close)(HPyContext ctx, HPy h);
@@ -21,6 +22,7 @@
     int (*ctx_Arg_Parse)(HPyContext ctx, HPy *args, HPy_ssize_t nargs, const char *fmt, va_list _vl);
     HPy (*ctx_Number_Add)(HPyContext ctx, HPy x, HPy y);
     HPy (*ctx_Unicode_FromString)(HPyContext ctx, const char *utf8);
+    void (*ctx_Err_SetString)(HPyContext ctx, HPy type, const char *message);
     HPy (*ctx_FromPyObject)(HPyContext ctx, struct _object *obj);
     struct _object *(*ctx_AsPyObject)(HPyContext ctx, HPy h);
     struct _object *(*ctx_CallRealFunctionFromTrampoline)(HPyContext ctx, struct _object *self, struct _object *args, void *func, int ml_flags);
diff --git a/pypy/module/hpy_universal/_vendored/include/universal/autogen_func.h b/pypy/module/hpy_universal/_vendored/include/universal/autogen_func.h
--- a/pypy/module/hpy_universal/_vendored/include/universal/autogen_func.h
+++ b/pypy/module/hpy_universal/_vendored/include/universal/autogen_func.h
@@ -40,6 +40,10 @@
      return ctx->ctx_Unicode_FromString ( ctx, utf8 ); 
 }
 
+static inline void HPyErr_SetString(HPyContext ctx, HPy type, const char *message) {
+     ctx->ctx_Err_SetString ( ctx, type, message ); 
+}
+
 static inline HPy HPy_FromPyObject(HPyContext ctx, struct _object *obj) {
      return ctx->ctx_FromPyObject ( ctx, obj ); 
 }
diff --git a/pypy/module/hpy_universal/_vendored/test/test_basic.py b/pypy/module/hpy_universal/_vendored/test/test_basic.py
--- a/pypy/module/hpy_universal/_vendored/test/test_basic.py
+++ b/pypy/module/hpy_universal/_vendored/test/test_basic.py
@@ -158,3 +158,26 @@
         """)
         assert mod.f(4) is False
         assert mod.f(6) is True
+
+    def test_exception(self):
+        import pytest
+        mod = self.make_module("""
+            HPy_METH_O(f)
+            static HPy f_impl(HPyContext ctx, HPy self, HPy arg)
+            {
+                long x = HPyLong_AsLong(ctx, arg);
+                if (x < 5) {
+                    return HPyLong_FromLong(ctx, -x);
+                }
+                else {
+                    HPyErr_SetString(ctx, ctx->h_ValueError, "hello world");
+                    return HPy_NULL;
+                }
+            }
+            @EXPORT f METH_O
+            @INIT
+        """)
+        assert mod.f(-10) == 10
+        with pytest.raises(ValueError) as exc:
+            mod.f(20)
+        assert str(exc.value) == 'hello world'

From pypy.commits at gmail.com  Mon Nov 18 12:08:55 2019
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 18 Nov 2019 09:08:55 -0800 (PST)
Subject: [pypy-commit] pypy hpy: (antocuni,
 arigo) WIP implementing test_exception
Message-ID: <5dd2d027.1c69fb81.9f190.dfe2@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98116:120fbccc3422
Date: 2019-11-18 18:07 +0100
http://bitbucket.org/pypy/pypy/changeset/120fbccc3422/

Log:	(antocuni, arigo) WIP implementing test_exception

	this mostly works already. However, if you raise an exception which
	is raised inside an llhelper callback, ll2ctypes raises it and
	ctypes prints it to stderr (by invoking sys.excepthook). After the
	printing, it is correctly propagated and thus the test passes.

	Next commits will try to convince ll2ctypes to do the right thing
	and just propagate the exception

diff --git a/pypy/module/hpy_universal/handles.py b/pypy/module/hpy_universal/handles.py
--- a/pypy/module/hpy_universal/handles.py
+++ b/pypy/module/hpy_universal/handles.py
@@ -4,6 +4,7 @@
     ('None', lambda space: space.w_None),
     ('False', lambda space: space.w_False),
     ('True', lambda space: space.w_True),
+    ('ValueError', lambda space: space.w_ValueError),
     ]
 
 
diff --git a/pypy/module/hpy_universal/interp_hpy.py b/pypy/module/hpy_universal/interp_hpy.py
--- a/pypy/module/hpy_universal/interp_hpy.py
+++ b/pypy/module/hpy_universal/interp_hpy.py
@@ -7,6 +7,7 @@
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.error import raise_import_error
 from pypy.interpreter.module import Module
+from pypy.interpreter.error import OperationError
 
 from pypy.module.hpy_universal import llapi, handles, interp_extfunc
 from pypy.module.hpy_universal.state import State
@@ -80,13 +81,23 @@
 
 @apifunc([llapi.HPyContext, rffi.CCHARP], llapi.HPy, error=0)
 def HPyUnicode_FromString(space, ctx, utf8):
+    w_obj = _maybe_utf8_to_w(space, utf8)
+    return handles.new(space, w_obj)
+
+def _maybe_utf8_to_w(space, utf8):
+    # should this be a method of space?
     s = rffi.charp2str(utf8)
     try:
         length = rutf8.check_utf8(s, allow_surrogates=False)
     except rutf8.CheckError:
         raise   # XXX do something
-    w_obj = space.newtext(s, length)
-    return handles.new(space, w_obj)
+    return space.newtext(s, length)
+
+ at apifunc([llapi.HPyContext, llapi.HPy, rffi.CCHARP], lltype.Void, error=None)
+def HPyErr_SetString(space, ctx, h_exc_type, utf8):
+   w_obj = _maybe_utf8_to_w(space, utf8)
+   w_exc_type = handles.deref(space, h_exc_type)
+   raise OperationError(w_exc_type, w_obj)
 
 
 def create_hpy_module(space, name, origin, lib, initfunc):
diff --git a/pypy/module/hpy_universal/llapi.py b/pypy/module/hpy_universal/llapi.py
--- a/pypy/module/hpy_universal/llapi.py
+++ b/pypy/module/hpy_universal/llapi.py
@@ -26,6 +26,7 @@
     ('h_None', HPy),
     ('h_True', HPy),
     ('h_False', HPy),
+    ('h_ValueError', HPy),
     ('ctx_Module_Create', rffi.VOIDP),
     ('ctx_Dup', rffi.VOIDP),
     ('ctx_Close', rffi.VOIDP),
@@ -34,6 +35,7 @@
     ('ctx_Arg_Parse', rffi.VOIDP),
     ('ctx_Number_Add', rffi.VOIDP),
     ('ctx_Unicode_FromString', rffi.VOIDP),
+    ('ctx_Err_SetString', rffi.VOIDP),
     ('ctx_FromPyObject', rffi.VOIDP),
     ('ctx_AsPyObject', rffi.VOIDP),
     ('ctx_CallRealFunctionFromTrampoline', rffi.VOIDP),
diff --git a/pypy/module/hpy_universal/state.py b/pypy/module/hpy_universal/state.py
--- a/pypy/module/hpy_universal/state.py
+++ b/pypy/module/hpy_universal/state.py
@@ -69,3 +69,6 @@
         #
         funcptr = interp_hpy.HPyUnicode_FromString.get_llhelper(space)
         self.ctx.c_ctx_Unicode_FromString = rffi.cast(rffi.VOIDP, funcptr)
+        #
+        funcptr = interp_hpy.HPyErr_SetString.get_llhelper(space)
+        self.ctx.c_ctx_Err_SetString = rffi.cast(rffi.VOIDP, funcptr)

From pypy.commits at gmail.com  Mon Nov 18 12:39:50 2019
From: pypy.commits at gmail.com (arigo)
Date: Mon, 18 Nov 2019 09:39:50 -0800 (PST)
Subject: [pypy-commit] pypy hpy: (antocuni, arigo)
Message-ID: <5dd2d766.1c69fb81.12808.ed2d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: hpy
Changeset: r98117:1b05295469eb
Date: 2019-11-18 18:38 +0100
http://bitbucket.org/pypy/pypy/changeset/1b05295469eb/

Log:	(antocuni, arigo)

	Allow llhelper functions to propagate exceptions from RPython to
	RPython via C. This already works after translation, but we need to
	convince ll2ctypes.

diff --git a/rpython/rtyper/lltypesystem/ll2ctypes.py b/rpython/rtyper/lltypesystem/ll2ctypes.py
--- a/rpython/rtyper/lltypesystem/ll2ctypes.py
+++ b/rpython/rtyper/lltypesystem/ll2ctypes.py
@@ -869,6 +869,9 @@
                             llinterp = LLInterpreter.current_interpreter
                             llinterp._store_exception(lle)
                             return 0
+                    return ctypes_return_value(llres)
+
+                def ctypes_return_value(llres):
                     assert lltype.typeOf(llres) == T.TO.RESULT
                     if T.TO.RESULT is lltype.Void:
                         return None
@@ -891,7 +894,12 @@
                         #    import pdb; pdb.post_mortem(sys.exc_traceback)
                         global _callback_exc_info
                         _callback_exc_info = sys.exc_info()
-                        raise
+                        if hasattr(getattr(container, '_callable', None),
+                                   '_llhelper_can_raise_'):
+                            llres = T.TO.RESULT._defl()
+                            return ctypes_return_value(llres)
+                        else:
+                            raise
 
                 if isinstance(T.TO.RESULT, lltype.Ptr):
                     TMod = lltype.Ptr(lltype.FuncType(T.TO.ARGS,
@@ -974,7 +982,7 @@
 
         return llobj
 
-def ctypes2lltype(T, cobj):
+def ctypes2lltype(T, cobj, force_real_ctypes_function=False):
     """Convert the ctypes object 'cobj' to its lltype equivalent.
     'T' is the expected lltype type.
     """
@@ -1045,7 +1053,7 @@
                 # or goes out of scope, then we crash.  CTypes is fun.
                 # It works if we cast it now to an int and back.
                 cobjkey = intmask(ctypes.cast(cobj, ctypes.c_void_p).value)
-                if cobjkey in _int2obj:
+                if cobjkey in _int2obj and not force_real_ctypes_function:
                     container = _int2obj[cobjkey]
                 else:
                     name = getattr(cobj, '__name__', '?')
diff --git a/rpython/rtyper/lltypesystem/test/test_ll2ctypes.py b/rpython/rtyper/lltypesystem/test/test_ll2ctypes.py
--- a/rpython/rtyper/lltypesystem/test/test_ll2ctypes.py
+++ b/rpython/rtyper/lltypesystem/test/test_ll2ctypes.py
@@ -1,4 +1,4 @@
-import py
+import py, pytest
 import sys, struct
 import ctypes
 from rpython.rtyper.lltypesystem import lltype, rffi, llmemory
@@ -480,6 +480,28 @@
         assert res == 42
         assert not ALLOCATED     # detects memory leaks in the test
 
+    def test_funcptr_can_raise(self, monkeypatch):
+        class FooError(Exception):
+            pass
+        def dummy(n):
+            raise FooError(n + 2)
+        dummy._llhelper_can_raise_ = True
+
+        FUNCTYPE = lltype.FuncType([lltype.Signed], lltype.Signed)
+        cdummy = lltype2ctypes(llhelper(lltype.Ptr(FUNCTYPE), dummy))
+        # here we pretend there is C in the middle
+        lldummy = ctypes2lltype(lltype.Ptr(FUNCTYPE), cdummy,
+                                force_real_ctypes_function=True)
+        seen = []
+        def custom_except_hook(*args):
+            seen.append(args)
+        monkeypatch.setattr(sys, 'excepthook', custom_except_hook)
+        with pytest.raises(FooError) as exc:
+            lldummy(41)
+        assert exc.value.args == (41 + 2,)
+        assert not seen
+        assert not ALLOCATED     # detects memory leaks in the test
+
     def test_funcptr2(self):
         FUNCTYPE = lltype.FuncType([rffi.CCHARP], rffi.LONG)
         cstrlen = standard_c_lib.strlen

From pypy.commits at gmail.com  Mon Nov 18 12:39:52 2019
From: pypy.commits at gmail.com (arigo)
Date: Mon, 18 Nov 2019 09:39:52 -0800 (PST)
Subject: [pypy-commit] pypy hpy: merge heads
Message-ID: <5dd2d768.1c69fb81.3fad.19c1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: hpy
Changeset: r98118:387736249e65
Date: 2019-11-18 18:39 +0100
http://bitbucket.org/pypy/pypy/changeset/387736249e65/

Log:	merge heads

diff --git a/pypy/module/hpy_universal/_vendored/include/cpython/hpy.h b/pypy/module/hpy_universal/_vendored/include/cpython/hpy.h
--- a/pypy/module/hpy_universal/_vendored/include/cpython/hpy.h
+++ b/pypy/module/hpy_universal/_vendored/include/cpython/hpy.h
@@ -37,23 +37,30 @@
     HPy h_None;
     HPy h_True;
     HPy h_False;
+    HPy h_ValueError;
 } *HPyContext;
 
 /* XXX! should be defined only once, not once for every .c! */
-static struct _HPyContext_s _global_ctx = {
-    .h_None = _py2h(Py_None),
-    .h_True = _py2h(Py_True),
-    .h_False = _py2h(Py_False),
-};
+static struct _HPyContext_s _global_ctx;
+
+#define HPy_NULL ((HPy){NULL})
+#define HPy_IsNull(x) ((x)._o == NULL)
 
 HPyAPI_FUNC(HPyContext)
 _HPyGetContext(void) {
-    return &_global_ctx;
+    HPyContext ctx = &_global_ctx;
+    if (HPy_IsNull(ctx->h_None)) {
+        // XXX: we need to find a better way to check whether the ctx is
+        // initialized or not
+        ctx->h_None = _py2h(Py_None);
+        ctx->h_True = _py2h(Py_True);
+        ctx->h_False = _py2h(Py_False);
+        ctx->h_ValueError = _py2h(PyExc_ValueError);
+    }
+    return ctx;
 }
 
 
-#define HPy_NULL ((HPy){NULL})
-#define HPy_IsNull(x) ((x)._o == NULL)
 
 HPyAPI_FUNC(HPy)
 HPyNone_Get(HPyContext ctx)
@@ -197,5 +204,11 @@
     return result;
 }
 
+HPyAPI_FUNC(void)
+HPyErr_SetString(HPyContext ctx, HPy type, const char *message)
+{
+    PyErr_SetString(_h2py(type), message);
+}
+
 
 #endif /* !HPy_CPYTHON_H */
diff --git a/pypy/module/hpy_universal/_vendored/include/universal/autogen_ctx.h b/pypy/module/hpy_universal/_vendored/include/universal/autogen_ctx.h
--- a/pypy/module/hpy_universal/_vendored/include/universal/autogen_ctx.h
+++ b/pypy/module/hpy_universal/_vendored/include/universal/autogen_ctx.h
@@ -13,6 +13,7 @@
     HPy h_None;
     HPy h_True;
     HPy h_False;
+    HPy h_ValueError;
     HPy (*ctx_Module_Create)(HPyContext ctx, HPyModuleDef *def);
     HPy (*ctx_Dup)(HPyContext ctx, HPy h);
     void (*ctx_Close)(HPyContext ctx, HPy h);
@@ -21,6 +22,7 @@
     int (*ctx_Arg_Parse)(HPyContext ctx, HPy *args, HPy_ssize_t nargs, const char *fmt, va_list _vl);
     HPy (*ctx_Number_Add)(HPyContext ctx, HPy x, HPy y);
     HPy (*ctx_Unicode_FromString)(HPyContext ctx, const char *utf8);
+    void (*ctx_Err_SetString)(HPyContext ctx, HPy type, const char *message);
     HPy (*ctx_FromPyObject)(HPyContext ctx, struct _object *obj);
     struct _object *(*ctx_AsPyObject)(HPyContext ctx, HPy h);
     struct _object *(*ctx_CallRealFunctionFromTrampoline)(HPyContext ctx, struct _object *self, struct _object *args, void *func, int ml_flags);
diff --git a/pypy/module/hpy_universal/_vendored/include/universal/autogen_func.h b/pypy/module/hpy_universal/_vendored/include/universal/autogen_func.h
--- a/pypy/module/hpy_universal/_vendored/include/universal/autogen_func.h
+++ b/pypy/module/hpy_universal/_vendored/include/universal/autogen_func.h
@@ -40,6 +40,10 @@
      return ctx->ctx_Unicode_FromString ( ctx, utf8 ); 
 }
 
+static inline void HPyErr_SetString(HPyContext ctx, HPy type, const char *message) {
+     ctx->ctx_Err_SetString ( ctx, type, message ); 
+}
+
 static inline HPy HPy_FromPyObject(HPyContext ctx, struct _object *obj) {
      return ctx->ctx_FromPyObject ( ctx, obj ); 
 }
diff --git a/pypy/module/hpy_universal/_vendored/test/test_basic.py b/pypy/module/hpy_universal/_vendored/test/test_basic.py
--- a/pypy/module/hpy_universal/_vendored/test/test_basic.py
+++ b/pypy/module/hpy_universal/_vendored/test/test_basic.py
@@ -158,3 +158,26 @@
         """)
         assert mod.f(4) is False
         assert mod.f(6) is True
+
+    def test_exception(self):
+        import pytest
+        mod = self.make_module("""
+            HPy_METH_O(f)
+            static HPy f_impl(HPyContext ctx, HPy self, HPy arg)
+            {
+                long x = HPyLong_AsLong(ctx, arg);
+                if (x < 5) {
+                    return HPyLong_FromLong(ctx, -x);
+                }
+                else {
+                    HPyErr_SetString(ctx, ctx->h_ValueError, "hello world");
+                    return HPy_NULL;
+                }
+            }
+            @EXPORT f METH_O
+            @INIT
+        """)
+        assert mod.f(-10) == 10
+        with pytest.raises(ValueError) as exc:
+            mod.f(20)
+        assert str(exc.value) == 'hello world'
diff --git a/pypy/module/hpy_universal/handles.py b/pypy/module/hpy_universal/handles.py
--- a/pypy/module/hpy_universal/handles.py
+++ b/pypy/module/hpy_universal/handles.py
@@ -4,6 +4,7 @@
     ('None', lambda space: space.w_None),
     ('False', lambda space: space.w_False),
     ('True', lambda space: space.w_True),
+    ('ValueError', lambda space: space.w_ValueError),
     ]
 
 
diff --git a/pypy/module/hpy_universal/interp_hpy.py b/pypy/module/hpy_universal/interp_hpy.py
--- a/pypy/module/hpy_universal/interp_hpy.py
+++ b/pypy/module/hpy_universal/interp_hpy.py
@@ -7,6 +7,7 @@
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.error import raise_import_error
 from pypy.interpreter.module import Module
+from pypy.interpreter.error import OperationError
 
 from pypy.module.hpy_universal import llapi, handles, interp_extfunc
 from pypy.module.hpy_universal.state import State
@@ -80,13 +81,23 @@
 
 @apifunc([llapi.HPyContext, rffi.CCHARP], llapi.HPy, error=0)
 def HPyUnicode_FromString(space, ctx, utf8):
+    w_obj = _maybe_utf8_to_w(space, utf8)
+    return handles.new(space, w_obj)
+
+def _maybe_utf8_to_w(space, utf8):
+    # should this be a method of space?
     s = rffi.charp2str(utf8)
     try:
         length = rutf8.check_utf8(s, allow_surrogates=False)
     except rutf8.CheckError:
         raise   # XXX do something
-    w_obj = space.newtext(s, length)
-    return handles.new(space, w_obj)
+    return space.newtext(s, length)
+
+ at apifunc([llapi.HPyContext, llapi.HPy, rffi.CCHARP], lltype.Void, error=None)
+def HPyErr_SetString(space, ctx, h_exc_type, utf8):
+   w_obj = _maybe_utf8_to_w(space, utf8)
+   w_exc_type = handles.deref(space, h_exc_type)
+   raise OperationError(w_exc_type, w_obj)
 
 
 def create_hpy_module(space, name, origin, lib, initfunc):
diff --git a/pypy/module/hpy_universal/llapi.py b/pypy/module/hpy_universal/llapi.py
--- a/pypy/module/hpy_universal/llapi.py
+++ b/pypy/module/hpy_universal/llapi.py
@@ -26,6 +26,7 @@
     ('h_None', HPy),
     ('h_True', HPy),
     ('h_False', HPy),
+    ('h_ValueError', HPy),
     ('ctx_Module_Create', rffi.VOIDP),
     ('ctx_Dup', rffi.VOIDP),
     ('ctx_Close', rffi.VOIDP),
@@ -34,6 +35,7 @@
     ('ctx_Arg_Parse', rffi.VOIDP),
     ('ctx_Number_Add', rffi.VOIDP),
     ('ctx_Unicode_FromString', rffi.VOIDP),
+    ('ctx_Err_SetString', rffi.VOIDP),
     ('ctx_FromPyObject', rffi.VOIDP),
     ('ctx_AsPyObject', rffi.VOIDP),
     ('ctx_CallRealFunctionFromTrampoline', rffi.VOIDP),
diff --git a/pypy/module/hpy_universal/state.py b/pypy/module/hpy_universal/state.py
--- a/pypy/module/hpy_universal/state.py
+++ b/pypy/module/hpy_universal/state.py
@@ -69,3 +69,6 @@
         #
         funcptr = interp_hpy.HPyUnicode_FromString.get_llhelper(space)
         self.ctx.c_ctx_Unicode_FromString = rffi.cast(rffi.VOIDP, funcptr)
+        #
+        funcptr = interp_hpy.HPyErr_SetString.get_llhelper(space)
+        self.ctx.c_ctx_Err_SetString = rffi.cast(rffi.VOIDP, funcptr)

From pypy.commits at gmail.com  Mon Nov 18 17:35:57 2019
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 18 Nov 2019 14:35:57 -0800 (PST)
Subject: [pypy-commit] pypy hpy: introduce a new decorator
 @llhelper_can_raise and use it to fix test_exception
Message-ID: <5dd31ccd.1c69fb81.27350.c21a@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98120:87553600e78a
Date: 2019-11-18 23:24 +0100
http://bitbucket.org/pypy/pypy/changeset/87553600e78a/

Log:	introduce a new decorator @llhelper_can_raise and use it to fix
	test_exception

diff --git a/pypy/module/hpy_universal/interp_hpy.py b/pypy/module/hpy_universal/interp_hpy.py
--- a/pypy/module/hpy_universal/interp_hpy.py
+++ b/pypy/module/hpy_universal/interp_hpy.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib.rdynload import dlopen, dlsym, DLOpenError
-from rpython.rlib.objectmodel import specialize
+from rpython.rlib.objectmodel import specialize, llhelper_can_raise
 from rpython.rlib import rutf8
 
 from pypy.interpreter.gateway import unwrap_spec
@@ -21,6 +21,7 @@
         ll_functype = lltype.Ptr(lltype.FuncType(argtypes, restype))
         @specialize.memo()
         def make_wrapper(space):
+            @llhelper_can_raise
             def wrapper(*args):
                 return fn(space, *args)
             return wrapper
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -226,6 +226,13 @@
     func._not_rpython_ = True
     return func
 
+def llhelper_can_raise(func):
+    """
+    Instruct ll2ctypes that this llhelper can raise RPython exceptions, which
+    should be propagated.
+    """
+    func._llhelper_can_raise_ = True
+    return func
 
 # ____________________________________________________________
 

From pypy.commits at gmail.com  Mon Nov 18 17:35:59 2019
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 18 Nov 2019 14:35:59 -0800 (PST)
Subject: [pypy-commit] pypy hpy: remove the error argument from @apifunc. It
 was unused and we decided to use a different approach for exceptions anyway
Message-ID: <5dd31ccf.1c69fb81.88268.5213@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98121:5c9a3fd99fc7
Date: 2019-11-18 23:28 +0100
http://bitbucket.org/pypy/pypy/changeset/5c9a3fd99fc7/

Log:	remove the error argument from @apifunc. It was unused and we
	decided to use a different approach for exceptions anyway

diff --git a/pypy/module/hpy_universal/interp_hpy.py b/pypy/module/hpy_universal/interp_hpy.py
--- a/pypy/module/hpy_universal/interp_hpy.py
+++ b/pypy/module/hpy_universal/interp_hpy.py
@@ -14,9 +14,7 @@
 from pypy.module.cpyext.api import generic_cpy_call_dont_convert_result
 
 
-def apifunc(argtypes, restype, error):
-    # XXX: at the moment, error is ignored. We should do something with it
-    # and handle exceptions properly
+def apifunc(argtypes, restype):
     def decorate(fn):
         ll_functype = lltype.Ptr(lltype.FuncType(argtypes, restype))
         @specialize.memo()
@@ -32,8 +30,7 @@
     return decorate
 
 
- at apifunc([llapi.HPyContext, lltype.Ptr(llapi.HPyModuleDef)],
-         llapi.HPy, error=0)
+ at apifunc([llapi.HPyContext, lltype.Ptr(llapi.HPyModuleDef)], llapi.HPy)
 def HPyModule_Create(space, ctx, hpydef):
     modname = rffi.charp2str(hpydef.c_m_name)
     w_mod = Module(space, space.newtext(modname))
@@ -50,20 +47,20 @@
     return handles.new(space, w_mod)
 
 
- at apifunc([llapi.HPyContext, llapi.HPy], llapi.HPy, error=0)
+ at apifunc([llapi.HPyContext, llapi.HPy], llapi.HPy)
 def HPy_Dup(space, ctx, h):
     return handles.dup(space, h)
 
- at apifunc([llapi.HPyContext, llapi.HPy], lltype.Void, error=None)
+ at apifunc([llapi.HPyContext, llapi.HPy], lltype.Void)
 def HPy_Close(space, ctx, h):
     handles.close(space, h)
 
- at apifunc([llapi.HPyContext, rffi.LONG], llapi.HPy, error=0)
+ at apifunc([llapi.HPyContext, rffi.LONG], llapi.HPy)
 def HPyLong_FromLong(space, ctx, value):
     w_obj = space.newint(rffi.cast(lltype.Signed, value))
     return handles.new(space, w_obj)
 
- at apifunc([llapi.HPyContext, llapi.HPy], rffi.LONG, error=0)
+ at apifunc([llapi.HPyContext, llapi.HPy], rffi.LONG)
 def HPyLong_AsLong(space, ctx, h):
     w_obj = handles.deref(space, h)
     #w_obj = space.int(w_obj)     --- XXX write a test for this
@@ -73,14 +70,14 @@
     #    ...
     return result
 
- at apifunc([llapi.HPyContext, llapi.HPy, llapi.HPy], llapi.HPy, error=0)
+ at apifunc([llapi.HPyContext, llapi.HPy, llapi.HPy], llapi.HPy)
 def HPyNumber_Add(space, ctx, h1, h2):
     w_obj1 = handles.deref(space, h1)
     w_obj2 = handles.deref(space, h2)
     w_result = space.add(w_obj1, w_obj2)
     return handles.new(space, w_result)
 
- at apifunc([llapi.HPyContext, rffi.CCHARP], llapi.HPy, error=0)
+ at apifunc([llapi.HPyContext, rffi.CCHARP], llapi.HPy)
 def HPyUnicode_FromString(space, ctx, utf8):
     w_obj = _maybe_utf8_to_w(space, utf8)
     return handles.new(space, w_obj)
@@ -94,7 +91,7 @@
         raise   # XXX do something
     return space.newtext(s, length)
 
- at apifunc([llapi.HPyContext, llapi.HPy, rffi.CCHARP], lltype.Void, error=None)
+ at apifunc([llapi.HPyContext, llapi.HPy, rffi.CCHARP], lltype.Void)
 def HPyErr_SetString(space, ctx, h_exc_type, utf8):
    w_obj = _maybe_utf8_to_w(space, utf8)
    w_exc_type = handles.deref(space, h_exc_type)

From pypy.commits at gmail.com  Tue Nov 19 04:46:27 2019
From: pypy.commits at gmail.com (arigo)
Date: Tue, 19 Nov 2019 01:46:27 -0800 (PST)
Subject: [pypy-commit] pypy default: Rename 'tuple_new' to '_PyPy_tuple_new'
 to follow the naming convention of
Message-ID: <5dd3b9f3.1c69fb81.619ce.b0f9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r98122:dcc4efe7355e
Date: 2019-11-19 10:45 +0100
http://bitbucket.org/pypy/pypy/changeset/dcc4efe7355e/

Log:	Rename 'tuple_new' to '_PyPy_tuple_new' to follow the naming
	convention of exported symbols in libpypy-c.so.

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1195,7 +1195,7 @@
         '_PyPy_get_PyOS_InputHook', [], FUNCPTR,
         compilation_info=eci, _nowrapper=True)
     state.C.tuple_new = rffi.llexternal(
-        'tuple_new', [PyTypeObjectPtr, PyObject, PyObject], PyObject,
+        '_PyPy_tuple_new', [PyTypeObjectPtr, PyObject, PyObject], PyObject,
         compilation_info=eci, _nowrapper=True)
 
 def init_function(func):
diff --git a/pypy/module/cpyext/include/tupleobject.h b/pypy/module/cpyext/include/tupleobject.h
--- a/pypy/module/cpyext/include/tupleobject.h
+++ b/pypy/module/cpyext/include/tupleobject.h
@@ -18,7 +18,7 @@
 
 PyAPI_FUNC(PyObject *) PyTuple_New(Py_ssize_t size);
 PyAPI_FUNC(void) _PyPy_tuple_dealloc(PyObject *);
-PyAPI_FUNC(PyObject *) tuple_new(PyTypeObject *type, PyObject *args, PyObject *kwds);
+PyAPI_FUNC(PyObject *) _PyPy_tuple_new(PyTypeObject *type, PyObject *args, PyObject *kwds);
 
 /* defined in varargswrapper.c */
 PyAPI_FUNC(PyObject *) PyTuple_Pack(Py_ssize_t, ...);
diff --git a/pypy/module/cpyext/src/tupleobject.c b/pypy/module/cpyext/src/tupleobject.c
--- a/pypy/module/cpyext/src/tupleobject.c
+++ b/pypy/module/cpyext/src/tupleobject.c
@@ -94,7 +94,7 @@
 tuple_subtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds);
 
 PyObject *
-tuple_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
+_PyPy_tuple_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
 {
     PyObject *arg = NULL;
     static char *kwlist[] = {"sequence", 0};
@@ -117,7 +117,7 @@
     Py_ssize_t i, n;
 
     assert(PyType_IsSubtype(type, &PyTuple_Type));
-    tmp = tuple_new(&PyTuple_Type, args, kwds);
+    tmp = _PyPy_tuple_new(&PyTuple_Type, args, kwds);
     if (tmp == NULL)
         return NULL;
     assert(PyTuple_Check(tmp));

From pypy.commits at gmail.com  Tue Nov 19 06:25:57 2019
From: pypy.commits at gmail.com (antocuni)
Date: Tue, 19 Nov 2019 03:25:57 -0800 (PST)
Subject: [pypy-commit] pypy hpy: This checkin might win the prize for the
 highest amount of XXXs/lines of code:
Message-ID: <5dd3d145.1c69fb81.8aee5.cdc9@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98123:f9c2cfcfaafa
Date: 2019-11-19 11:48 +0100
http://bitbucket.org/pypy/pypy/changeset/f9c2cfcfaafa/

Log:	This checkin might win the prize for the highest amount of
	XXXs/lines of code: it needs a deep review, please :)

	Fix W_ExtensionFunction_call_varargs to use the updated calling
	convention, and implement ctx_Arg_Parse.

diff --git a/pypy/module/hpy_universal/interp_extfunc.py b/pypy/module/hpy_universal/interp_extfunc.py
--- a/pypy/module/hpy_universal/interp_extfunc.py
+++ b/pypy/module/hpy_universal/interp_extfunc.py
@@ -47,9 +47,33 @@
         return handles.consume(space, h_result)
 
     def call_varargs(self, space, arguments_w):
-        w_tuple = space.newtuple(arguments_w)
-        # xxx here we just invoke call_o() with the w_tuple
-        return self.call_o(space, w_tuple)
+        # this function is more or less the equivalent of
+        # ctx_CallRealFunctionFromTrampoline in cpython-universal
+        state = space.fromcache(State)
+        n = len(arguments_w)
+
+        # XXX this looks inefficient: ideally, we would like the equivalent of
+        # alloca(): do we have it in RPython? The alternative is to wrap
+        # arguments_w in a tuple, convert to handle and pass it to a C
+        # function whichs calls alloca() and the forwards everything to the
+        # functpr
+        with handles.using(space, self.w_self) as h_self:
+            with lltype.scoped_alloc(rffi.CArray(llapi.HPy), n) as args_h:
+                for i, w_obj in enumerate(arguments_w):
+                    args_h[i] = handles.new(space, w_obj)
+
+                # XXX: is it correct to use rffi.cast instead of some kind of
+                # lltype.cast_*?
+                fptr = rffi.cast(llapi.HPyMeth_VarArgs, self.cfuncptr)
+                h_result = generic_cpy_call_dont_convert_result(space, fptr,
+                                                        state.ctx, h_self, args_h, n)
+
+                # XXX this should probably be in a try/finally. We should add a
+                # test to check that we don't leak handles
+                for i in range(n):
+                    handles.close(space, args_h[i])
+
+        return handles.consume(space, h_result)
 
     def descr_call(self, space, __args__):
         flags = self.flags
diff --git a/pypy/module/hpy_universal/llapi.py b/pypy/module/hpy_universal/llapi.py
--- a/pypy/module/hpy_universal/llapi.py
+++ b/pypy/module/hpy_universal/llapi.py
@@ -1,23 +1,37 @@
 import py
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
+from pypy import pypydir
 from pypy.module.hpy_universal import _vendored
 
-INCLUDE_DIR = str(py.path.local(_vendored.__file__).dirpath().join('include'))
+PYPYDIR = py.path.local(pypydir)
+INCLUDE_DIR = PYPYDIR.join('module', 'hpy_universal', '_vendored', 'include')
+SRC_DIR = PYPYDIR.join('module', 'hpy_universal', 'src')
+
+# XXX I don't understand what is going on here: if I put getargs.c as
+# separate_module_files, then ll2ctypes can't find it. I need to #include t in
+# separate_module_sources for now...
 eci = ExternalCompilationInfo(includes=["universal/hpy.h"],
                               include_dirs=[INCLUDE_DIR],
+                              ## separate_module_files=[
+                              ##     SRC_DIR.join('getargs.c'),
+                              ##     ],
                               post_include_bits=["""
 RPY_EXTERN void *_HPy_GetGlobalCtx(void);
+RPY_EXTERN int ctx_Arg_Parse(HPyContext ctx, HPy *args, HPy_ssize_t nargs,
+                             const char *fmt, va_list vl);
 """],
                               separate_module_sources=["""
 
+#include "%s"
+
 struct _HPyContext_s hpy_global_ctx;
 void *_HPy_GetGlobalCtx(void)
 {
     return &hpy_global_ctx;
 }
 
-"""])
+""" % SRC_DIR.join('getargs.c')])
 
 
 HPy = lltype.Signed
@@ -43,11 +57,17 @@
 )
 HPyContext = lltype.Ptr(HPyContextS)
 
+HPy_ssize_t = lltype.Signed # XXXXXXXXX?
+
 HPyInitFuncPtr = lltype.Ptr(lltype.FuncType([HPyContext], HPy))
 
 _HPyCFunctionPtr = lltype.Ptr(lltype.FuncType([HPyContext, HPy, HPy], HPy))
 _HPy_CPyCFunctionPtr = rffi.VOIDP    # not used here
 
+HPyMeth_VarArgs = lltype.Ptr(
+    lltype.FuncType([HPyContext, HPy, lltype.Ptr(rffi.CArray(HPy)), HPy_ssize_t], HPy))
+
+
 _HPyMethodPairFuncPtr = lltype.Ptr(lltype.FuncType([
         rffi.CArrayPtr(_HPyCFunctionPtr),
         rffi.CArrayPtr(_HPy_CPyCFunctionPtr)],
@@ -76,8 +96,18 @@
 METH_O        = 0x0008
 
 
+
+
 # ----------------------------------------------------------------
 
 
 _HPy_GetGlobalCtx = rffi.llexternal('_HPy_GetGlobalCtx', [], HPyContext,
                                     compilation_info=eci, _nowrapper=True)
+
+# NOTE: this is not the real signature (we don't know what to put for
+# va_list), but it's good enough to get the address of the function to store
+# in the ctx. DO NOT CALL THIS!. TO avoid possible mistakes, we directly cast
+# it to VOIDP
+ctx_Arg_Parse_fn = rffi.llexternal('ctx_Arg_Parse', [], rffi.INT_real,
+                                compilation_info=eci, _nowrapper=True)
+ctx_Arg_Parse = rffi.cast(rffi.VOIDP, ctx_Arg_Parse_fn)
diff --git a/pypy/module/hpy_universal/src/getargs.c b/pypy/module/hpy_universal/src/getargs.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/src/getargs.c
@@ -0,0 +1,43 @@
+#include "universal/hpy.h"
+
+#include <stdio.h>
+
+/* XXX: this function is copied&pasted THREE times:
+ *     hpy_devel/include/hpy.h
+ *     cpython-universal/api.c
+ *     pypy/module/hpy_universal/src/getargs.c
+ *
+ * We need a way to share this kind of common code
+ */
+
+/* would be nice if this were static :( */
+int
+ctx_Arg_Parse(HPyContext ctx, HPy *args, HPy_ssize_t nargs,
+              const char *fmt, va_list vl)
+{
+    const char *fmt1 = fmt;
+    HPy_ssize_t i = 0;
+
+    while (*fmt1 != 0) {
+        if (i >= nargs) {
+            abort(); // XXX
+        }
+        switch (*fmt1++) {
+        case 'l': {
+            long *output = va_arg(vl, long *);
+            long value = HPyLong_AsLong(ctx, args[i]);
+            // XXX check for exceptions
+            *output = value;
+            break;
+        }
+        default:
+            abort();  // XXX
+        }
+        i++;
+    }
+    if (i != nargs) {
+        abort();   // XXX
+    }
+
+    return 1;
+}
diff --git a/pypy/module/hpy_universal/state.py b/pypy/module/hpy_universal/state.py
--- a/pypy/module/hpy_universal/state.py
+++ b/pypy/module/hpy_universal/state.py
@@ -72,3 +72,5 @@
         #
         funcptr = interp_hpy.HPyErr_SetString.get_llhelper(space)
         self.ctx.c_ctx_Err_SetString = rffi.cast(rffi.VOIDP, funcptr)
+        #
+        self.ctx.c_ctx_Arg_Parse = llapi.ctx_Arg_Parse

From pypy.commits at gmail.com  Tue Nov 19 12:21:48 2019
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 19 Nov 2019 09:21:48 -0800 (PST)
Subject: [pypy-commit] pypy py3.6: Remove obsolete test
Message-ID: <5dd424ac.1c69fb81.df6a0.14c1@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6
Changeset: r98124:fc3e8b80874c
Date: 2019-11-19 17:20 +0000
http://bitbucket.org/pypy/pypy/changeset/fc3e8b80874c/

Log:	Remove obsolete test

diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -91,40 +91,18 @@
         except Exception as exc:
             e = exc
             import sys
-            exc_type,exc_val,tb = sys.exc_info()
+            exc_type, exc_val, tb = sys.exc_info()
         try:
             raise Exception   # 6 lines below the previous one
         except Exception as exc:
             e2 = exc
-            exc_type2,exc_val2,tb2 = sys.exc_info()
+            exc_type2, exc_val2, tb2 = sys.exc_info()
         assert exc_type ==Exception
         assert exc_val ==e
         assert exc_type2 ==Exception
         assert exc_val2 ==e2
         assert tb2.tb_lineno - tb.tb_lineno == 6
 
-    def test_dynamic_attributes(self):
-        try:
-            raise Exception
-        except Exception as exc:
-            e = exc
-            import sys
-            exc_type = sys.exc_type
-            exc_val = sys.exc_value
-            tb = sys.exc_traceback
-        try:
-            raise Exception   # 8 lines below the previous one
-        except Exception as exc:
-            e2 = exc
-            exc_type2 = sys.exc_type
-            exc_val2 = sys.exc_value
-            tb2 = sys.exc_traceback
-        assert exc_type ==Exception
-        assert exc_val ==e
-        assert exc_type2 ==Exception
-        assert exc_val2 ==e2
-        assert tb2.tb_lineno - tb.tb_lineno == 8
-
     def test_exc_info_normalization(self):
         import sys
         try:

From pypy.commits at gmail.com  Thu Nov 21 09:47:27 2019
From: pypy.commits at gmail.com (Stian Andreassen)
Date: Thu, 21 Nov 2019 06:47:27 -0800 (PST)
Subject: [pypy-commit] pypy default: Fix issue #3120,
 do restore traceback in PyErr_Restore.
Message-ID: <5dd6a37f.1c69fb81.8bbd9.901e@mx.google.com>

Author: Stian Andreassen
Branch: 
Changeset: r98125:3b12b6935197
Date: 2019-11-21 15:46 +0100
http://bitbucket.org/pypy/pypy/changeset/3b12b6935197/

Log:	Fix issue #3120, do restore traceback in PyErr_Restore.

diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -88,7 +88,7 @@
     if w_type is None:
         state.clear_exception()
         return
-    state.set_exception(OperationError(w_type, w_value))
+    state.set_exception(OperationError(w_type, w_value, w_traceback))
 
 @cpython_api([PyObjectP, PyObjectP, PyObjectP], lltype.Void)
 def PyErr_NormalizeException(space, exc_p, val_p, tb_p):

From pypy.commits at gmail.com  Thu Nov 21 09:52:23 2019
From: pypy.commits at gmail.com (antocuni)
Date: Thu, 21 Nov 2019 06:52:23 -0800 (PST)
Subject: [pypy-commit] pypy hpy: extend checkmodule to be able to invoke the
 C compiler if requested. Use the new functionality to test hpy_universal
 translation,
 which currently fails during the C compilation step due to a misdeclared
 type in the HPyContextS Struct. Will be fixed in next commits
Message-ID: <5dd6a4a7.1c69fb81.bb526.1c0a@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98126:c427e9f459bf
Date: 2019-11-21 15:51 +0100
http://bitbucket.org/pypy/pypy/changeset/c427e9f459bf/

Log:	extend checkmodule to be able to invoke the C compiler if requested.
	Use the new functionality to test hpy_universal translation, which
	currently fails during the C compilation step due to a misdeclared
	type in the HPyContextS Struct. Will be fixed in next commits

diff --git a/pypy/module/hpy_universal/test/test_ztranslation.py b/pypy/module/hpy_universal/test/test_ztranslation.py
--- a/pypy/module/hpy_universal/test/test_ztranslation.py
+++ b/pypy/module/hpy_universal/test/test_ztranslation.py
@@ -1,6 +1,17 @@
-
+from rpython.translator.c.test.test_standalone import StandaloneTests
+from rpython.config.translationoption import get_combined_translation_config
+from pypy.config.pypyoption import get_pypy_config
 from pypy.objspace.fake.checkmodule import checkmodule
+from pypy.objspace.fake.objspace import FakeObjSpace
+from pypy.module.hpy_universal.state import State
 
 def test_checkmodule():
-    checkmodule('hpy_universal')
+    def extra_func(space):
+        state = space.fromcache(State)
+        state.setup()
 
+    config_opts = {'translation.gc': 'boehm'}
+    checkmodule('hpy_universal',
+                extra_func=extra_func,
+                c_compile=True,
+                config_opts=config_opts)
diff --git a/pypy/objspace/fake/checkmodule.py b/pypy/objspace/fake/checkmodule.py
--- a/pypy/objspace/fake/checkmodule.py
+++ b/pypy/objspace/fake/checkmodule.py
@@ -2,14 +2,29 @@
 from pypy.config.pypyoption import get_pypy_config
 
 
-def checkmodule(*modnames, **kwds):
-    translate_startup = kwds.pop('translate_startup', True)
-    ignore = set(kwds.pop('ignore', ()))
-    assert not kwds
+def checkmodule(modname, translate_startup=True, ignore=(),
+                c_compile=False, extra_func=None, config_opts=None):
+    """
+    Check that the module 'modname' translates.
+
+    Options:
+      translate_startup: TODO, document me
+
+      ignore: list of module interpleveldefs/appleveldefs to ignore
+
+      c_compile: determine whether to inokve the C compiler after rtyping
+
+      extra_func: extra function which will be annotated and called. It takes
+                  a single "space" argment
+
+      config_opts: dictionary containing extra configuration options which
+                   will be passed to TranslationContext
+    """
     config = get_pypy_config(translating=True)
     space = FakeObjSpace(config)
     seeobj_w = []
     modules = []
+    modnames = [modname]
     for modname in modnames:
         mod = __import__(
             'pypy.module.%s.moduledef' % modname, None, None, ['__doc__'])
@@ -34,5 +49,8 @@
     if not translate_startup:
         func()   # call it now
         func = None
+
+    opts = {'translation.list_comprehension_operations': True}
+    opts.update(config_opts)
     space.translates(func, seeobj_w=seeobj_w,
-                     **{'translation.list_comprehension_operations': True})
+                     c_compile=c_compile, extra_func=extra_func, **opts)
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -1,4 +1,5 @@
 from rpython.annotator.model import SomeInstance, s_None
+from rpython.annotator.listdef import s_list_of_strings
 from rpython.rlib.objectmodel import (instantiate, we_are_translated, specialize,
     not_rpython)
 from rpython.rlib.nonconst import NonConstant
@@ -7,6 +8,7 @@
 from rpython.rtyper.lltypesystem import lltype
 from rpython.tool.sourcetools import compile2, func_with_new_name
 from rpython.translator.translator import TranslationContext
+from rpython.translator.c.genc import CStandaloneBuilder
 
 from pypy.tool.option import make_config
 from pypy.interpreter import argument, gateway
@@ -387,7 +389,9 @@
 
     # ----------
 
-    def translates(self, func=None, argtypes=None, seeobj_w=[], **kwds):
+    def translates(self, func=None, argtypes=None, seeobj_w=[],
+                   extra_func=None, c_compile=False,
+                   **kwds):
         config = make_config(None, **kwds)
         if func is not None:
             if argtypes is None:
@@ -397,10 +401,14 @@
         t = TranslationContext(config=config)
         self.t = t     # for debugging
         ann = t.buildannotator()
-        def _do_startup():
+
+        def entry_point(argv):
             self.threadlocals.enter_thread(self)
             W_SliceObject(w_some_obj(), w_some_obj(), w_some_obj())
-        ann.build_types(_do_startup, [], complete_now=False)
+            if extra_func:
+                extra_func(self)
+            return 0
+        ann.build_types(entry_point, [s_list_of_strings], complete_now=False)
         if func is not None:
             ann.build_types(func, argtypes, complete_now=False)
         if seeobj_w:
@@ -422,6 +430,12 @@
         #t.viewcg()
         t.buildrtyper().specialize()
         t.checkgraphs()
+        if c_compile:
+            cbuilder = CStandaloneBuilder(t, entry_point, t.config)
+            cbuilder.generate_source(defines=cbuilder.DEBUG_DEFINES)
+            cbuilder.compile()
+            return t, cbuilder
+
 
     def setup(space):
         for name in (ObjSpace.ConstantTable +

From pypy.commits at gmail.com  Thu Nov 21 10:28:49 2019
From: pypy.commits at gmail.com (antocuni)
Date: Thu, 21 Nov 2019 07:28:49 -0800 (PST)
Subject: [pypy-commit] pypy hpy: fix test_handles,
 which now requires a more capable fake space
Message-ID: <5dd6ad31.1c69fb81.1591d.1d3b@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98127:b848790a2adb
Date: 2019-11-21 16:06 +0100
http://bitbucket.org/pypy/pypy/changeset/b848790a2adb/

Log:	fix test_handles, which now requires a more capable fake space

diff --git a/pypy/module/hpy_universal/test/test_handles.py b/pypy/module/hpy_universal/test/test_handles.py
--- a/pypy/module/hpy_universal/test/test_handles.py
+++ b/pypy/module/hpy_universal/test/test_handles.py
@@ -1,38 +1,61 @@
+import pytest
 from pypy.module.hpy_universal import handles
 from pypy.module.hpy_universal.handles import HandleManager
 
+class FakeSpace(object):
+    def __init__(self):
+        self._cache = {}
+    
+    def fromcache(self, cls):
+        if cls not in self._cache:
+            self._cache[cls] = cls(self)
+        return self._cache[cls]
+
+    def __getattr__(self, name):
+        return '<fakespace.%s>' % name
+
+ at pytest.fixture
+def fakespace():
+    return FakeSpace()
+
+def test_fakespace(fakespace):
+    assert fakespace.w_ValueError == '<fakespace.w_ValueError>'
+    def x(space):
+        return object()
+    assert fakespace.fromcache(x) is fakespace.fromcache(x)
+
 class TestHandleManager(object):
 
-    def test_first_handle_is_not_zero(self):
-        mgr = HandleManager(None)
+    def test_first_handle_is_not_zero(self, fakespace):
+        mgr = HandleManager(fakespace)
         h = mgr.new('hello')
         assert h > 0
 
-    def test_new(self):
-        mgr = HandleManager(None)
+    def test_new(self, fakespace):
+        mgr = HandleManager(fakespace)
         h = mgr.new('hello')
         assert mgr.handles_w[h] == 'hello'
 
-    def test_close(self):
-        mgr = HandleManager(None)
+    def test_close(self, fakespace):
+        mgr = HandleManager(fakespace)
         h = mgr.new('hello')
         assert mgr.close(h) is None
         assert mgr.handles_w[h] is None
 
-    def test_deref(self):
-        mgr = HandleManager(None)
+    def test_deref(self, fakespace):
+        mgr = HandleManager(fakespace)
         h = mgr.new('hello')
         assert mgr.deref(h) == 'hello'     # 'hello' is a fake W_Root
         assert mgr.deref(h) == 'hello'
 
-    def test_consume(self):
-        mgr = HandleManager(None)
+    def test_consume(self, fakespace):
+        mgr = HandleManager(fakespace)
         h = mgr.new('hello')
         assert mgr.consume(h) == 'hello'
         assert mgr.handles_w[h] is None
 
-    def test_freelist(self):
-        mgr = HandleManager(None)
+    def test_freelist(self, fakespace):
+        mgr = HandleManager(fakespace)
         h0 = mgr.new('hello')
         h1 = mgr.new('world')
         assert mgr.consume(h0) == 'hello'
@@ -41,20 +64,15 @@
         assert h2 == h0
         assert mgr.free_list == []
 
-    def test_dup(self):
-        mgr = HandleManager(None)
+    def test_dup(self, fakespace):
+        mgr = HandleManager(fakespace)
         h0 = mgr.new('hello')
         h1 = mgr.dup(h0)
         assert h1 != h0
         assert mgr.consume(h0) == mgr.consume(h1) == 'hello'
 
-def test_using():
-    mgr = HandleManager(None)
-    class FakeSpace(object):
-        @classmethod
-        def fromcache(cls, x):
-            return mgr
-    space = FakeSpace()
-    with handles.using(space, 'hello') as h:
+def test_using(fakespace):
+    mgr = fakespace.fromcache(HandleManager)
+    with handles.using(fakespace, 'hello') as h:
         assert mgr.handles_w[h] == 'hello'
     assert mgr.handles_w[h] is None

From pypy.commits at gmail.com  Thu Nov 21 10:28:51 2019
From: pypy.commits at gmail.com (antocuni)
Date: Thu, 21 Nov 2019 07:28:51 -0800 (PST)
Subject: [pypy-commit] pypy hpy: introduce HPyS_real,
 which is the actualy C struct used to represent handles in C,
 and use it in the definition of HPyContextS. This fixes test_ztranslation,
 and hopefully the real translation as well
Message-ID: <5dd6ad33.1c69fb81.5b722.ff39@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98128:60cbdc3bb88a
Date: 2019-11-21 16:28 +0100
http://bitbucket.org/pypy/pypy/changeset/60cbdc3bb88a/

Log:	introduce HPyS_real, which is the actualy C struct used to represent
	handles in C, and use it in the definition of HPyContextS. This
	fixes test_ztranslation, and hopefully the real translation as well

diff --git a/pypy/module/hpy_universal/llapi.py b/pypy/module/hpy_universal/llapi.py
--- a/pypy/module/hpy_universal/llapi.py
+++ b/pypy/module/hpy_universal/llapi.py
@@ -33,14 +33,25 @@
 
 """ % SRC_DIR.join('getargs.c')])
 
+HPy_ssize_t = lltype.Signed # XXXXXXXXX?
 
-HPy = lltype.Signed
+# for practical reason, we use a primitive type to represent HPy almost
+# everywhere in RPython. HOWEVER, the "real" HPy C type which is defined in
+# universal/hpy.h is an anonymous struct: we need to use it e.g. to represent
+# fields inside HPyContextS
+HPy = HPy_ssize_t
+HPyS_real = rffi.CStruct('HPy',
+    ('_i', HPy_ssize_t),
+    hints={'eci': eci, 'typedef': True},
+)
+
+
 HPyContextS = rffi.CStruct('_HPyContext_s',
     ('ctx_version', rffi.INT_real),
-    ('h_None', HPy),
-    ('h_True', HPy),
-    ('h_False', HPy),
-    ('h_ValueError', HPy),
+    ('h_None', HPyS_real),
+    ('h_True', HPyS_real),
+    ('h_False', HPyS_real),
+    ('h_ValueError', HPyS_real),
     ('ctx_Module_Create', rffi.VOIDP),
     ('ctx_Dup', rffi.VOIDP),
     ('ctx_Close', rffi.VOIDP),
@@ -56,9 +67,6 @@
     hints={'eci': eci},
 )
 HPyContext = lltype.Ptr(HPyContextS)
-
-HPy_ssize_t = lltype.Signed # XXXXXXXXX?
-
 HPyInitFuncPtr = lltype.Ptr(lltype.FuncType([HPyContext], HPy))
 
 _HPyCFunctionPtr = lltype.Ptr(lltype.FuncType([HPyContext, HPy, HPy], HPy))
diff --git a/pypy/module/hpy_universal/state.py b/pypy/module/hpy_universal/state.py
--- a/pypy/module/hpy_universal/state.py
+++ b/pypy/module/hpy_universal/state.py
@@ -6,7 +6,7 @@
 from rpython.rlib.objectmodel import specialize
 from pypy.module.hpy_universal import llapi, handles
 
-CONTEXT_NAMES = unrolling_iterable(llapi.HPyContext.TO._names)
+CONTEXT_FIELDS = unrolling_iterable(llapi.HPyContext.TO._names)
 CONSTANT_NAMES = unrolling_iterable([name for name, _ in handles.CONSTANTS])
 DUMMY_FUNC = lltype.FuncType([], lltype.Void)
 
@@ -33,17 +33,21 @@
         space = self.space
         self.ctx = llapi._HPy_GetGlobalCtx()
 
-        for name in CONTEXT_NAMES:
+        for name in CONTEXT_FIELDS:
             if name == 'c_ctx_version':
                 continue
             if name.startswith('c_ctx_'):
+                # this is a function pointer: assign a default value so we get
+                # a reasonable error message if it's called without being
+                # assigned to something else
                 missing_function = make_missing_function(name)
                 funcptr = llhelper(lltype.Ptr(DUMMY_FUNC), missing_function)
                 setattr(self.ctx, name, rffi.cast(rffi.VOIDP, funcptr))
         i = 0
         for name in CONSTANT_NAMES:
             if name != 'NULL':
-                setattr(self.ctx, 'c_h_' + name, i)
+                h_struct = getattr(self.ctx, 'c_h_' + name)
+                h_struct.c__i = i
             i = i + 1
 
         # XXX collect all these functions automatically

From pypy.commits at gmail.com  Thu Nov 21 10:32:17 2019
From: pypy.commits at gmail.com (Stian Andreassen)
Date: Thu, 21 Nov 2019 07:32:17 -0800 (PST)
Subject: [pypy-commit] pypy default: Remove the XXX about w_traceback
Message-ID: <5dd6ae01.1c69fb81.ae765.3438@mx.google.com>

Author: Stian Andreassen
Branch: 
Changeset: r98129:2916cb12db62
Date: 2019-11-21 16:31 +0100
http://bitbucket.org/pypy/pypy/changeset/2916cb12db62/

Log:	Remove the XXX about w_traceback

diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -84,7 +84,7 @@
     w_type = get_w_obj_and_decref(space, py_type)
     w_value = get_w_obj_and_decref(space, py_value)
     w_traceback = get_w_obj_and_decref(space, py_traceback)
-    # XXX do something with w_traceback
+
     if w_type is None:
         state.clear_exception()
         return

From pypy.commits at gmail.com  Thu Nov 21 10:33:01 2019
From: pypy.commits at gmail.com (stevie_92)
Date: Thu, 21 Nov 2019 07:33:01 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-gc-cycle: Implemented incremental limit
 for non-rrc objects during rrc marking
Message-ID: <5dd6ae2d.1c69fb81.f159a.2bed@mx.google.com>

Author: Stefan Beyer <home at sbeyer.at>
Branch: cpyext-gc-cycle
Changeset: r98130:6a71ef8d2b2c
Date: 2019-11-21 15:50 +0100
http://bitbucket.org/pypy/pypy/changeset/6a71ef8d2b2c/

Log:	Implemented incremental limit for non-rrc objects during rrc marking
	Added flag to enable/disable tuple untracking for debugging

diff --git a/rpython/memory/gc/rrc/base.py b/rpython/memory/gc/rrc/base.py
--- a/rpython/memory/gc/rrc/base.py
+++ b/rpython/memory/gc/rrc/base.py
@@ -84,6 +84,8 @@
     RAWREFCOUNT_REFS_UNTRACKED = -2 << RAWREFCOUNT_REFS_SHIFT
     RAWREFCOUNT_REFS_REACHABLE = -3 << RAWREFCOUNT_REFS_SHIFT
 
+    UNTRACK_TUPLES_DEFAULT = True
+
     def _pyobj(self, pyobjaddr):
         return llmemory.cast_adr_to_ptr(pyobjaddr, self.PYOBJ_HDR_PTR)
     def _pygchdr(self, pygchdraddr):
@@ -124,6 +126,13 @@
         self.state = self.STATE_DEFAULT
         self.marking_state = 0
         inc_limit = env.read_uint_from_env('PYPY_RRC_GC_INCREMENT_STEP')
+        untrack = env.read_uint_from_env('PYPY_RRC_GC_UNTRACK_TUPLES')
+        if untrack == 1:
+            self.untrack_tuples = True
+        elif untrack == 2:
+            self.untrack_tuples = False
+        else:
+            self.untrack_tuples = self.UNTRACK_TUPLES_DEFAULT
         if inc_limit > 0:
             self.inc_limit = inc_limit
         else:
@@ -469,17 +478,20 @@
             self._free(pyobject, True)
 
     def _untrack_tuples(self):
-        gchdr = self.tuple_list.c_gc_next
-        while gchdr <> self.tuple_list:
-            gchdr_next = gchdr.c_gc_next
-            pyobj = self.gc_as_pyobj(gchdr)
-            result = self.tuple_maybe_untrack(pyobj)
-            if result == 1: # contains gc objects -> promote to pyobj list
-                next = gchdr.c_gc_next
-                next.c_gc_prev = gchdr.c_gc_prev
-                gchdr.c_gc_prev.c_gc_next = next
-                self._gc_list_add(self.pyobj_list, gchdr)
-            gchdr = gchdr_next
+        if self.untrack_tuples:
+            self._debug_check_consistency(print_label="before-untrack")
+            gchdr = self.tuple_list.c_gc_next
+            while gchdr <> self.tuple_list:
+                gchdr_next = gchdr.c_gc_next
+                pyobj = self.gc_as_pyobj(gchdr)
+                result = self.tuple_maybe_untrack(pyobj)
+                if result == 1: # contains gc objects -> promote to pyobj list
+                    next = gchdr.c_gc_next
+                    next.c_gc_prev = gchdr.c_gc_prev
+                    gchdr.c_gc_prev.c_gc_next = next
+                    self._gc_list_add(self.pyobj_list, gchdr)
+                gchdr = gchdr_next
+            self._debug_check_consistency(print_label="after-untrack")
 
     def _find_garbage(self, use_dict):
         found_garbage = False
diff --git a/rpython/memory/gc/rrc/incmark.py b/rpython/memory/gc/rrc/incmark.py
--- a/rpython/memory/gc/rrc/incmark.py
+++ b/rpython/memory/gc/rrc/incmark.py
@@ -2,6 +2,7 @@
 from rpython.rtyper.lltypesystem import rffi
 from rpython.memory.gc.rrc.base import RawRefCountBaseGC
 from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop
+from rpython.rlib.rarithmetic import intmask
 import time
 
 class RawRefCountIncMarkGC(RawRefCountBaseGC):
@@ -367,9 +368,22 @@
                 snapobj.refcnt += 1
                 self._mark_rawrefcount_obj(snapobj)
                 simple_limit += 1
-                if simple_limit > self.inc_limit: # TODO: add test
+                if simple_limit > intmask(self.inc_limit): # TODO: add test
                     reached_limit = True
-            self.gc.visit_all_objects()  # TODO: implement sane limit
+
+            if not reached_limit:
+                # "split" the limit between rrc and non-rrc
+                # if 25% of rrc limit have been used, now use max 75% or non-rrc limit
+                if simple_limit > 0:
+                    estimate = intmask(self.gc.gc_increment_step) * intmask(self.inc_limit) / simple_limit
+                else:
+                    estimate = intmask(self.gc.gc_increment_step)
+                remaining = self.gc.visit_all_objects_step(estimate)
+                if remaining == 0:
+                    reached_limit = True
+                else:
+                    # if 25% of the non-rrc limit have been used, add 25% of the rrc limit to the counter
+                    simple_limit += intmask(self.inc_limit) * remaining / estimate
             first = False
         return not reached_limit # are there any objects left?
 
diff --git a/rpython/memory/gc/rrc/simple.py b/rpython/memory/gc/rrc/simple.py
--- a/rpython/memory/gc/rrc/simple.py
+++ b/rpython/memory/gc/rrc/simple.py
@@ -2,6 +2,12 @@
 
 class RawRefCountSimpleGC(RawRefCountBaseGC):
 
+    UNTRACK_TUPLES_DEFAULT = False
+
     def major_collection_trace_step(self):
+        self._untrack_tuples()
         self.p_list_old.foreach(self._major_trace, (False, False))
-        return True
\ No newline at end of file
+        return True
+
+    def visit_pyobj(self, gcobj):
+        pass
\ No newline at end of file

From pypy.commits at gmail.com  Thu Nov 21 12:16:11 2019
From: pypy.commits at gmail.com (Stian Andreassen)
Date: Thu, 21 Nov 2019 09:16:11 -0800 (PST)
Subject: [pypy-commit] pypy default: The test for my previous commit
Message-ID: <5dd6c65b.1c69fb81.ae87e.1e9f@mx.google.com>

Author: Stian Andreassen
Branch: 
Changeset: r98131:f49f7df7c633
Date: 2019-11-21 18:15 +0100
http://bitbucket.org/pypy/pypy/changeset/f49f7df7c633/

Log:	The test for my previous commit

diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py
--- a/pypy/module/cpyext/test/test_pyerrors.py
+++ b/pypy/module/cpyext/test/test_pyerrors.py
@@ -3,6 +3,7 @@
 import StringIO
 
 from pypy.module.cpyext.state import State
+from pypy.module.cpyext.pyobject import make_ref
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from rpython.rtyper.lltypesystem import rffi
@@ -108,6 +109,30 @@
         assert recieved == ['ok']
         assert api.PyOS_InterruptOccurred()
 
+    def test_restore_traceback(self, space, api):
+        string = rffi.str2charp("spam and eggs")
+        api.PyErr_SetString(space.w_ValueError, string)
+
+        state = space.fromcache(State)
+        operror = state.clear_exception()
+
+        # Fake a traceback.
+        operror.set_traceback(space.w_True) # this doesn't really need to be a real traceback for this test.
+
+        w_type = operror.w_type
+        w_value = operror.get_w_value(space)
+        w_tb = operror.get_w_traceback(space)
+
+        assert not space.eq_w(w_tb, space.w_None)
+
+        api.PyErr_Restore(make_ref(space, w_type), make_ref(space, w_value), make_ref(space, w_tb))
+
+        operror = state.clear_exception()
+        w_tb_restored = operror.get_w_traceback(space)
+
+        assert space.eq_w(w_tb_restored, w_tb)
+        rffi.free_charp(string)
+
 class AppTestFetch(AppTestCpythonExtensionBase):
 
     def test_occurred(self):

From pypy.commits at gmail.com  Thu Nov 21 12:39:09 2019
From: pypy.commits at gmail.com (antocuni)
Date: Thu, 21 Nov 2019 09:39:09 -0800 (PST)
Subject: [pypy-commit] pypy hpy: silence a gcc warning
Message-ID: <5dd6cbbd.1c69fb81.7f222.271b@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98132:b4cd0dba403c
Date: 2019-11-21 18:27 +0100
http://bitbucket.org/pypy/pypy/changeset/b4cd0dba403c/

Log:	silence a gcc warning

diff --git a/pypy/module/hpy_universal/state.py b/pypy/module/hpy_universal/state.py
--- a/pypy/module/hpy_universal/state.py
+++ b/pypy/module/hpy_universal/state.py
@@ -77,4 +77,5 @@
         funcptr = interp_hpy.HPyErr_SetString.get_llhelper(space)
         self.ctx.c_ctx_Err_SetString = rffi.cast(rffi.VOIDP, funcptr)
         #
-        self.ctx.c_ctx_Arg_Parse = llapi.ctx_Arg_Parse
+        # the cast is not strictly necessary but silences a gcc warning
+        self.ctx.c_ctx_Arg_Parse = rffi.cast(rffi.VOIDP, llapi.ctx_Arg_Parse)

From pypy.commits at gmail.com  Thu Nov 21 12:39:11 2019
From: pypy.commits at gmail.com (antocuni)
Date: Thu, 21 Nov 2019 09:39:11 -0800 (PST)
Subject: [pypy-commit] pypy hpy-ctypespace: try to use cpyext's CTypeSpace
 to declare some of the types needed by hpy. Hopefully,
 this will make things clearer and potentially will reduce code duplication
 with pyhandle/hpy
Message-ID: <5dd6cbbf.1c69fb81.2e87f.ac46@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy-ctypespace
Changeset: r98133:fbcf93255972
Date: 2019-11-21 18:38 +0100
http://bitbucket.org/pypy/pypy/changeset/fbcf93255972/

Log:	try to use cpyext's CTypeSpace to declare some of the types needed
	by hpy. Hopefully, this will make things clearer and potentially
	will reduce code duplication with pyhandle/hpy

diff --git a/pypy/module/hpy_universal/llapi.py b/pypy/module/hpy_universal/llapi.py
--- a/pypy/module/hpy_universal/llapi.py
+++ b/pypy/module/hpy_universal/llapi.py
@@ -2,7 +2,7 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from pypy import pypydir
-from pypy.module.hpy_universal import _vendored
+from pypy.module.cpyext.cparser import CTypeSpace
 
 PYPYDIR = py.path.local(pypydir)
 INCLUDE_DIR = PYPYDIR.join('module', 'hpy_universal', '_vendored', 'include')
@@ -20,6 +20,10 @@
 RPY_EXTERN void *_HPy_GetGlobalCtx(void);
 RPY_EXTERN int ctx_Arg_Parse(HPyContext ctx, HPy *args, HPy_ssize_t nargs,
                              const char *fmt, va_list vl);
+
+// this is a workaround for a CTypeSpace limitation, since it can't properly
+handle struct types which are not typedefs
+typedef struct _HPyContext_s _struct_HPyContext_s;
 """],
                               separate_module_sources=["""
 
@@ -30,43 +34,51 @@
 {
     return &hpy_global_ctx;
 }
-
 """ % SRC_DIR.join('getargs.c')])
 
-HPy_ssize_t = lltype.Signed # XXXXXXXXX?
+cts = CTypeSpace()
+# NOTE: the following C source is NOT seen by the C compiler during
+# translation: it is used only as a nice way to declare the lltype.* types
+# which are needed here
+cts.headers.append('stdint.h')
+cts.parse_source("""
+typedef intptr_t HPy_ssize_t;
+typedef struct { HPy_ssize_t _i; } HPy;
+typedef struct _HPyContext_s {
+    int ctx_version;
+    HPy h_None;
+    HPy h_True;
+    HPy h_False;
+    HPy h_ValueError;
+    void *ctx_Module_Create;
+    void *ctx_Dup;
+    void *ctx_Close;
+    void *ctx_Long_FromLong;
+    void *ctx_Long_AsLong;
+    void *ctx_Arg_Parse;
+    void *ctx_Number_Add;
+    void *ctx_Unicode_FromString;
+    void *ctx_Err_SetString;
+    void *ctx_FromPyObject;
+    void *ctx_AsPyObject;
+    void *ctx_CallRealFunctionFromTrampoline;
+} _struct_HPyContext_s;
+typedef struct _HPyContext_s *HPyContext;
+""")
+
+HPy_ssize_t = cts.gettype('HPy_ssize_t')
+# XXX: HPyContext is equivalent to the old HPyContext which was defined
+# explicitly using rffi.CStruct: the only different is that this is missing
+# hints={'eci': eci}: however, the tests still pass (including
+# ztranslation). Why was the eci needed?
+HPyContext = cts.gettype('HPyContext')
 
 # for practical reason, we use a primitive type to represent HPy almost
-# everywhere in RPython. HOWEVER, the "real" HPy C type which is defined in
-# universal/hpy.h is an anonymous struct: we need to use it e.g. to represent
-# fields inside HPyContextS
+# everywhere in RPython. HOWEVER, the "real" HPy C type is a struct
 HPy = HPy_ssize_t
-HPyS_real = rffi.CStruct('HPy',
-    ('_i', HPy_ssize_t),
-    hints={'eci': eci, 'typedef': True},
-)
+_HPy_real = cts.gettype('HPy')
 
 
-HPyContextS = rffi.CStruct('_HPyContext_s',
-    ('ctx_version', rffi.INT_real),
-    ('h_None', HPyS_real),
-    ('h_True', HPyS_real),
-    ('h_False', HPyS_real),
-    ('h_ValueError', HPyS_real),
-    ('ctx_Module_Create', rffi.VOIDP),
-    ('ctx_Dup', rffi.VOIDP),
-    ('ctx_Close', rffi.VOIDP),
-    ('ctx_Long_FromLong', rffi.VOIDP),
-    ('ctx_Long_AsLong', rffi.VOIDP),
-    ('ctx_Arg_Parse', rffi.VOIDP),
-    ('ctx_Number_Add', rffi.VOIDP),
-    ('ctx_Unicode_FromString', rffi.VOIDP),
-    ('ctx_Err_SetString', rffi.VOIDP),
-    ('ctx_FromPyObject', rffi.VOIDP),
-    ('ctx_AsPyObject', rffi.VOIDP),
-    ('ctx_CallRealFunctionFromTrampoline', rffi.VOIDP),
-    hints={'eci': eci},
-)
-HPyContext = lltype.Ptr(HPyContextS)
 HPyInitFuncPtr = lltype.Ptr(lltype.FuncType([HPyContext], HPy))
 
 _HPyCFunctionPtr = lltype.Ptr(lltype.FuncType([HPyContext, HPy, HPy], HPy))

From pypy.commits at gmail.com  Thu Nov 21 12:57:24 2019
From: pypy.commits at gmail.com (antocuni)
Date: Thu, 21 Nov 2019 09:57:24 -0800 (PST)
Subject: [pypy-commit] pypy hpy-ctypespace: bah, typo
Message-ID: <5dd6d004.1c69fb81.ac20b.4c46@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy-ctypespace
Changeset: r98134:d788317a107f
Date: 2019-11-21 18:56 +0100
http://bitbucket.org/pypy/pypy/changeset/d788317a107f/

Log:	bah, typo

diff --git a/pypy/module/hpy_universal/llapi.py b/pypy/module/hpy_universal/llapi.py
--- a/pypy/module/hpy_universal/llapi.py
+++ b/pypy/module/hpy_universal/llapi.py
@@ -22,7 +22,7 @@
                              const char *fmt, va_list vl);
 
 // this is a workaround for a CTypeSpace limitation, since it can't properly
-handle struct types which are not typedefs
+// handle struct types which are not typedefs
 typedef struct _HPyContext_s _struct_HPyContext_s;
 """],
                               separate_module_sources=["""

From pypy.commits at gmail.com  Thu Nov 21 13:24:59 2019
From: pypy.commits at gmail.com (antocuni)
Date: Thu, 21 Nov 2019 10:24:59 -0800 (PST)
Subject: [pypy-commit] pypy hpy: I'm not sure exactly what's happening, but
 with the double cast we end up with a global array inside
 ctx->ctx_Arg_Parse; doing this seems to solve the issue
Message-ID: <5dd6d67b.1c69fb81.7f222.3525@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98135:160e0b8f64a1
Date: 2019-11-21 19:24 +0100
http://bitbucket.org/pypy/pypy/changeset/160e0b8f64a1/

Log:	I'm not sure exactly what's happening, but with the double cast we
	end up with a global array inside ctx->ctx_Arg_Parse; doing this
	seems to solve the issue

diff --git a/pypy/module/hpy_universal/llapi.py b/pypy/module/hpy_universal/llapi.py
--- a/pypy/module/hpy_universal/llapi.py
+++ b/pypy/module/hpy_universal/llapi.py
@@ -114,8 +114,6 @@
 
 # NOTE: this is not the real signature (we don't know what to put for
 # va_list), but it's good enough to get the address of the function to store
-# in the ctx. DO NOT CALL THIS!. TO avoid possible mistakes, we directly cast
-# it to VOIDP
-ctx_Arg_Parse_fn = rffi.llexternal('ctx_Arg_Parse', [], rffi.INT_real,
-                                compilation_info=eci, _nowrapper=True)
-ctx_Arg_Parse = rffi.cast(rffi.VOIDP, ctx_Arg_Parse_fn)
+# in the ctx. DO NOT CALL THIS!
+DONT_CALL_ctx_Arg_Parse = rffi.llexternal('ctx_Arg_Parse', [], rffi.INT_real,
+                                          compilation_info=eci, _nowrapper=True)
diff --git a/pypy/module/hpy_universal/state.py b/pypy/module/hpy_universal/state.py
--- a/pypy/module/hpy_universal/state.py
+++ b/pypy/module/hpy_universal/state.py
@@ -77,5 +77,4 @@
         funcptr = interp_hpy.HPyErr_SetString.get_llhelper(space)
         self.ctx.c_ctx_Err_SetString = rffi.cast(rffi.VOIDP, funcptr)
         #
-        # the cast is not strictly necessary but silences a gcc warning
-        self.ctx.c_ctx_Arg_Parse = rffi.cast(rffi.VOIDP, llapi.ctx_Arg_Parse)
+        self.ctx.c_ctx_Arg_Parse = rffi.cast(rffi.VOIDP, llapi.DONT_CALL_ctx_Arg_Parse)

From pypy.commits at gmail.com  Thu Nov 21 13:25:01 2019
From: pypy.commits at gmail.com (antocuni)
Date: Thu, 21 Nov 2019 10:25:01 -0800 (PST)
Subject: [pypy-commit] pypy hpy-ctypespace: merge from hpy
Message-ID: <5dd6d67d.1c69fb81.79ed7.6b12@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy-ctypespace
Changeset: r98136:f88d90878e29
Date: 2019-11-21 19:24 +0100
http://bitbucket.org/pypy/pypy/changeset/f88d90878e29/

Log:	merge from hpy

diff --git a/pypy/module/hpy_universal/llapi.py b/pypy/module/hpy_universal/llapi.py
--- a/pypy/module/hpy_universal/llapi.py
+++ b/pypy/module/hpy_universal/llapi.py
@@ -126,8 +126,6 @@
 
 # NOTE: this is not the real signature (we don't know what to put for
 # va_list), but it's good enough to get the address of the function to store
-# in the ctx. DO NOT CALL THIS!. TO avoid possible mistakes, we directly cast
-# it to VOIDP
-ctx_Arg_Parse_fn = rffi.llexternal('ctx_Arg_Parse', [], rffi.INT_real,
-                                compilation_info=eci, _nowrapper=True)
-ctx_Arg_Parse = rffi.cast(rffi.VOIDP, ctx_Arg_Parse_fn)
+# in the ctx. DO NOT CALL THIS!
+DONT_CALL_ctx_Arg_Parse = rffi.llexternal('ctx_Arg_Parse', [], rffi.INT_real,
+                                          compilation_info=eci, _nowrapper=True)
diff --git a/pypy/module/hpy_universal/state.py b/pypy/module/hpy_universal/state.py
--- a/pypy/module/hpy_universal/state.py
+++ b/pypy/module/hpy_universal/state.py
@@ -77,5 +77,4 @@
         funcptr = interp_hpy.HPyErr_SetString.get_llhelper(space)
         self.ctx.c_ctx_Err_SetString = rffi.cast(rffi.VOIDP, funcptr)
         #
-        # the cast is not strictly necessary but silences a gcc warning
-        self.ctx.c_ctx_Arg_Parse = rffi.cast(rffi.VOIDP, llapi.ctx_Arg_Parse)
+        self.ctx.c_ctx_Arg_Parse = rffi.cast(rffi.VOIDP, llapi.DONT_CALL_ctx_Arg_Parse)

From pypy.commits at gmail.com  Fri Nov 22 11:26:38 2019
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 22 Nov 2019 08:26:38 -0800 (PST)
Subject: [pypy-commit] pypy hpy-ctypespace: move the typefef HPyInitFunc
 into cts;
 I removed the 'Ptr' from it to follow CPython convention (e.g.,
 PyCFunction). In order to make it working, I had to move the HPy/_HPy_real
 distinction inside the cts itself
Message-ID: <5dd80c3e.1c69fb81.5f148.7e8c@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy-ctypespace
Changeset: r98138:c00a74b4ddb9
Date: 2019-11-21 19:37 +0100
http://bitbucket.org/pypy/pypy/changeset/c00a74b4ddb9/

Log:	move the typefef HPyInitFunc into cts; I removed the 'Ptr' from it
	to follow CPython convention (e.g., PyCFunction). In order to make
	it working, I had to move the HPy/_HPy_real distinction inside the
	cts itself

diff --git a/pypy/module/hpy_universal/interp_hpy.py b/pypy/module/hpy_universal/interp_hpy.py
--- a/pypy/module/hpy_universal/interp_hpy.py
+++ b/pypy/module/hpy_universal/interp_hpy.py
@@ -100,7 +100,7 @@
 
 def create_hpy_module(space, name, origin, lib, initfunc):
     state = space.fromcache(State)
-    initfunc = rffi.cast(llapi.HPyInitFuncPtr, initfunc)
+    initfunc = rffi.cast(llapi.HPyInitFunc, initfunc)
     h_module = generic_cpy_call_dont_convert_result(space, initfunc, state.ctx)
     return handles.consume(space, h_module)
 
diff --git a/pypy/module/hpy_universal/llapi.py b/pypy/module/hpy_universal/llapi.py
--- a/pypy/module/hpy_universal/llapi.py
+++ b/pypy/module/hpy_universal/llapi.py
@@ -43,13 +43,17 @@
 cts.headers.append('stdint.h')
 cts.parse_source("""
 typedef intptr_t HPy_ssize_t;
-typedef struct { HPy_ssize_t _i; } HPy;
+
+// see below for more info about HPy vs _HPy_real
+typedef HPy_ssize_t HPy;
+typedef struct { HPy_ssize_t _i; } _HPy_real;
+
 typedef struct _HPyContext_s {
     int ctx_version;
-    HPy h_None;
-    HPy h_True;
-    HPy h_False;
-    HPy h_ValueError;
+    _HPy_real h_None;
+    _HPy_real h_True;
+    _HPy_real h_False;
+    _HPy_real h_ValueError;
     void *ctx_Module_Create;
     void *ctx_Dup;
     void *ctx_Close;
@@ -64,6 +68,8 @@
     void *ctx_CallRealFunctionFromTrampoline;
 } _struct_HPyContext_s;
 typedef struct _HPyContext_s *HPyContext;
+
+typedef HPy (*HPyInitFunc)(HPyContext ctx);
 """)
 
 HPy_ssize_t = cts.gettype('HPy_ssize_t')
@@ -75,11 +81,11 @@
 
 # for practical reason, we use a primitive type to represent HPy almost
 # everywhere in RPython. HOWEVER, the "real" HPy C type is a struct
-HPy = HPy_ssize_t
-_HPy_real = cts.gettype('HPy')
+HPy = cts.gettype('HPy')
+_HPy_real = cts.gettype('_HPy_real')
 
 
-HPyInitFuncPtr = lltype.Ptr(lltype.FuncType([HPyContext], HPy))
+HPyInitFunc = cts.gettype('HPyInitFunc')
 
 _HPyCFunctionPtr = lltype.Ptr(lltype.FuncType([HPyContext, HPy, HPy], HPy))
 _HPy_CPyCFunctionPtr = rffi.VOIDP    # not used here

From pypy.commits at gmail.com  Fri Nov 22 11:26:40 2019
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 22 Nov 2019 08:26:40 -0800 (PST)
Subject: [pypy-commit] pypy hpy-ctypespace: move more types into cts
Message-ID: <5dd80c40.1c69fb81.a2cd4.4466@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy-ctypespace
Changeset: r98139:86cbe337bc16
Date: 2019-11-22 11:21 +0100
http://bitbucket.org/pypy/pypy/changeset/86cbe337bc16/

Log:	move more types into cts

diff --git a/pypy/module/hpy_universal/interp_extfunc.py b/pypy/module/hpy_universal/interp_extfunc.py
--- a/pypy/module/hpy_universal/interp_extfunc.py
+++ b/pypy/module/hpy_universal/interp_extfunc.py
@@ -19,10 +19,9 @@
         # fetch the real HPy function pointer, by calling ml_meth, which
         # is a function that returns it and also the CPython-only trampoline
         with lltype.scoped_alloc(
-                rffi.CArray(llapi._HPyCFunctionPtr), 1) as funcptr:
+                rffi.CArray(llapi._HPyCFunction), 1) as funcptr:
             with lltype.scoped_alloc(
-                    rffi.CArray(llapi._HPy_CPyCFunctionPtr), 1)  \
-                                                        as ignored_trampoline:
+                    rffi.CArray(llapi._HPyCPyCFunction), 1) as ignored_trampoline:
                 ml.c_ml_meth(funcptr, ignored_trampoline)
                 self.cfuncptr = funcptr[0]
 
diff --git a/pypy/module/hpy_universal/llapi.py b/pypy/module/hpy_universal/llapi.py
--- a/pypy/module/hpy_universal/llapi.py
+++ b/pypy/module/hpy_universal/llapi.py
@@ -70,6 +70,29 @@
 typedef struct _HPyContext_s *HPyContext;
 
 typedef HPy (*HPyInitFunc)(HPyContext ctx);
+
+typedef HPy (*_HPyCFunction)(HPyContext ctx, HPy self, HPy args);
+typedef void *_HPyCPyCFunction; // not used here
+typedef void (*_HPyMethodPairFunc)(_HPyCFunction *out_func,
+                                   _HPyCPyCFunction *out_trampoline);
+
+
+typedef struct {
+    const char         *ml_name;
+    _HPyMethodPairFunc ml_meth;
+    int                ml_flags;
+    const char         *ml_doc;
+} HPyMethodDef;
+
+typedef struct {
+    void *dummy; // this is needed because we put a comma after HPyModuleDef_HEAD_INIT :(
+    const char* m_name;
+    const char* m_doc;
+    HPy_ssize_t m_size;
+    HPyMethodDef *m_methods;
+} HPyModuleDef;
+
+
 """)
 
 HPy_ssize_t = cts.gettype('HPy_ssize_t')
@@ -84,37 +107,19 @@
 HPy = cts.gettype('HPy')
 _HPy_real = cts.gettype('_HPy_real')
 
-
 HPyInitFunc = cts.gettype('HPyInitFunc')
-
-_HPyCFunctionPtr = lltype.Ptr(lltype.FuncType([HPyContext, HPy, HPy], HPy))
-_HPy_CPyCFunctionPtr = rffi.VOIDP    # not used here
+_HPyCFunction = cts.gettype('_HPyCFunction')
+_HPyCPyCFunction = cts.gettype('_HPyCPyCFunction')
 
 HPyMeth_VarArgs = lltype.Ptr(
     lltype.FuncType([HPyContext, HPy, lltype.Ptr(rffi.CArray(HPy)), HPy_ssize_t], HPy))
 
-
-_HPyMethodPairFuncPtr = lltype.Ptr(lltype.FuncType([
-        rffi.CArrayPtr(_HPyCFunctionPtr),
-        rffi.CArrayPtr(_HPy_CPyCFunctionPtr)],
-    lltype.Void))
-
-HPyMethodDef = rffi.CStruct('HPyMethodDef',
-    ('ml_name', rffi.CCHARP),
-    ('ml_meth', _HPyMethodPairFuncPtr),
-    ('ml_flags', rffi.INT_real),
-    ('ml_doc', rffi.CCHARP),
-    hints={'eci': eci, 'typedef': True},
-)
-
-HPyModuleDef = rffi.CStruct('HPyModuleDef',
-    ('dummy', rffi.VOIDP),
-    ('m_name', rffi.CCHARP),
-    ('m_doc', rffi.CCHARP),
-    ('m_size', lltype.Signed),
-    ('m_methods', rffi.CArrayPtr(HPyMethodDef)),
-    hints={'eci': eci, 'typedef': True},
-)
+HPyMethodDef = cts.gettype('HPyMethodDef')
+HPyModuleDef = cts.gettype('HPyModuleDef')
+# CTypeSpace converts "HPyMethodDef*" into lltype.Ptr(HPyMethodDef), but we
+# want a CArrayPtr instead, so that we can index the items inside
+# HPyModule_Create
+HPyModuleDef._flds['c_m_methods'] = rffi.CArrayPtr(HPyMethodDef)
 
 METH_VARARGS  = 0x0001
 METH_KEYWORDS = 0x0002

From pypy.commits at gmail.com  Fri Nov 22 11:26:42 2019
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 22 Nov 2019 08:26:42 -0800 (PST)
Subject: [pypy-commit] pypy hpy-ctypespace: Introduce rffi.constcharp2str,
 and improve the repr of low-level array
Message-ID: <5dd80c42.1c69fb81.41033.948d@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy-ctypespace
Changeset: r98140:a8819c541390
Date: 2019-11-22 12:06 +0100
http://bitbucket.org/pypy/pypy/changeset/a8819c541390/

Log:	Introduce rffi.constcharp2str, and improve the repr of low-level
	array types.

	Previously, you got annotation errors like this, which were VERY
	confusing:

	 * UnionError:
	    - SomePtr(ll_ptrtype=<* Array of Char >)
	    - SomePtr(ll_ptrtype=<* Array of Char >)

	Now you get the much more meaningful:

	 * UnionError:
	    - SomePtr(ll_ptrtype=<* Array of Char {'nolength': True} >)
	    - SomePtr(ll_ptrtype=<* Array of Char {'render_as_const': True,
	'nolength': True} >)

diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py
--- a/rpython/rtyper/lltypesystem/lltype.py
+++ b/rpython/rtyper/lltypesystem/lltype.py
@@ -467,12 +467,16 @@
     _str_fields = saferecursive(_str_fields, '...')
 
     def __str__(self):
-        return "%s of %s " % (self.__class__.__name__,
-                               self._str_fields(),)
+        hints = str(self._hints) if self._hints else ''
+        return "%s of %s %s " % (self.__class__.__name__,
+                                 self._str_fields(),
+                                 hints)
 
     def _short_name(self):
-        return "%s %s" % (self.__class__.__name__,
-                          self.OF._short_name(),)
+        hints = str(self._hints) if self._hints else ''
+        return "%s %s %s" % (self.__class__.__name__,
+                             self.OF._short_name(),
+                             hints)
     _short_name = saferecursive(_short_name, '...')
 
     def _container_example(self):
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -1024,6 +1024,14 @@
  ) = make_string_mappings(unicode)
 
 
+def constcharp2str(cp):
+    """
+    Like charp2str, but takes a CONST_CCHARP instead
+    """
+    cp = cast(CCHARP, cp)
+    return charp2str(cp)
+constcharp2str._annenforceargs_ = [lltype.SomePtr(CONST_CCHARP)]
+
 @not_rpython
 def _deprecated_get_nonmovingbuffer(*args):
     raise Exception(
diff --git a/rpython/rtyper/lltypesystem/test/test_rffi.py b/rpython/rtyper/lltypesystem/test/test_rffi.py
--- a/rpython/rtyper/lltypesystem/test/test_rffi.py
+++ b/rpython/rtyper/lltypesystem/test/test_rffi.py
@@ -142,6 +142,33 @@
         xf = self.compile(f, [], backendopt=False)
         assert xf() == 3
 
+    def test_constcharp2str(self):
+        c_source = py.code.Source("""
+        const char *z(void)
+        {
+            return "hello world";
+        }
+        """)
+        eci = ExternalCompilationInfo(separate_module_sources=[c_source],
+                                     post_include_bits=['const char *z(void);'])
+        z = llexternal('z', [], CONST_CCHARP, compilation_info=eci)
+
+        def f():
+            l_buf = lltype.malloc(CCHARP.TO, 5, flavor='raw')
+            l_buf[0] = 'A'
+            l_buf[1] = 'B'
+            l_buf[2] = 'C'
+            l_buf[3] = '\x00'
+            l_buf[4] = 'E'
+            l_constbuf = cast(CONST_CCHARP, l_buf)
+            res = constcharp2str(l_constbuf)
+            lltype.free(l_buf, flavor='raw')
+            return len(res)
+
+        assert f() == 3
+        xf = self.compile(f, [], backendopt=False)
+        assert xf() == 3
+
     def test_stringstar(self):
         c_source = """
         #include <string.h>

From pypy.commits at gmail.com  Fri Nov 22 11:26:43 2019
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 22 Nov 2019 08:26:43 -0800 (PST)
Subject: [pypy-commit] pypy hpy-ctypespace: use the new rffi.constcharp2str
 to match the lltype of these fields
Message-ID: <5dd80c43.1c69fb81.61d64.9ebf@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy-ctypespace
Changeset: r98141:3915c72fffc4
Date: 2019-11-22 15:13 +0100
http://bitbucket.org/pypy/pypy/changeset/3915c72fffc4/

Log:	use the new rffi.constcharp2str to match the lltype of these fields

diff --git a/pypy/module/hpy_universal/interp_extfunc.py b/pypy/module/hpy_universal/interp_extfunc.py
--- a/pypy/module/hpy_universal/interp_extfunc.py
+++ b/pypy/module/hpy_universal/interp_extfunc.py
@@ -14,7 +14,7 @@
     def __init__(self, ml, w_self):
         self.ml = ml
         self.w_self = w_self
-        self.name = rffi.charp2str(self.ml.c_ml_name)
+        self.name = rffi.constcharp2str(self.ml.c_ml_name)
         self.flags = rffi.cast(lltype.Signed, self.ml.c_ml_flags)
         # fetch the real HPy function pointer, by calling ml_meth, which
         # is a function that returns it and also the CPython-only trampoline
diff --git a/pypy/module/hpy_universal/interp_hpy.py b/pypy/module/hpy_universal/interp_hpy.py
--- a/pypy/module/hpy_universal/interp_hpy.py
+++ b/pypy/module/hpy_universal/interp_hpy.py
@@ -32,7 +32,7 @@
 
 @apifunc([llapi.HPyContext, lltype.Ptr(llapi.HPyModuleDef)], llapi.HPy)
 def HPyModule_Create(space, ctx, hpydef):
-    modname = rffi.charp2str(hpydef.c_m_name)
+    modname = rffi.constcharp2str(hpydef.c_m_name)
     w_mod = Module(space, space.newtext(modname))
     #
     # add all the functions defined in hpydef.c_m_methods

From pypy.commits at gmail.com  Fri Nov 22 11:26:45 2019
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 22 Nov 2019 08:26:45 -0800 (PST)
Subject: [pypy-commit] pypy hpy-ctypespace: fix translation again: we need a
 way to convince gcc that our notion of _struct_HPy_s is the same as its
Message-ID: <5dd80c45.1c69fb81.eb8be.b03a@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy-ctypespace
Changeset: r98142:79350cb71de6
Date: 2019-11-22 17:16 +0100
http://bitbucket.org/pypy/pypy/changeset/79350cb71de6/

Log:	fix translation again: we need a way to convince gcc that our notion
	of _struct_HPy_s is the same as its

diff --git a/pypy/module/hpy_universal/_vendored/include/universal/hpy.h b/pypy/module/hpy_universal/_vendored/include/universal/hpy.h
--- a/pypy/module/hpy_universal/_vendored/include/universal/hpy.h
+++ b/pypy/module/hpy_universal/_vendored/include/universal/hpy.h
@@ -6,7 +6,11 @@
 #include <stdarg.h>
 
 typedef intptr_t HPy_ssize_t;
-typedef struct { HPy_ssize_t _i; } HPy;
+
+// WARNING: the following change has been done inside the pypy hpy-ctypespace
+// branch. If/when the branch is merged, we should backport it to pyhandle/hpy
+struct _HPy_s { HPy_ssize_t _i; };
+typedef struct _HPy_s HPy;
 
 typedef struct _HPyContext_s *HPyContext;
 struct _object;  /* that's PyObject inside CPython */
diff --git a/pypy/module/hpy_universal/llapi.py b/pypy/module/hpy_universal/llapi.py
--- a/pypy/module/hpy_universal/llapi.py
+++ b/pypy/module/hpy_universal/llapi.py
@@ -21,9 +21,11 @@
 RPY_EXTERN int ctx_Arg_Parse(HPyContext ctx, HPy *args, HPy_ssize_t nargs,
                              const char *fmt, va_list vl);
 
-// this is a workaround for a CTypeSpace limitation, since it can't properly
+
+// these are workarounds for a CTypeSpace limitation, since it can't properly
 // handle struct types which are not typedefs
 typedef struct _HPyContext_s _struct_HPyContext_s;
+typedef struct _HPy_s _struct_HPy_s;
 """],
                               separate_module_sources=["""
 
@@ -44,16 +46,18 @@
 cts.parse_source("""
 typedef intptr_t HPy_ssize_t;
 
-// see below for more info about HPy vs _HPy_real
+// see below for more info about HPy vs _struct_HPy_s
+typedef struct _HPy_s {
+    HPy_ssize_t _i;
+} _struct_HPy_s;
 typedef HPy_ssize_t HPy;
-typedef struct { HPy_ssize_t _i; } _HPy_real;
 
 typedef struct _HPyContext_s {
     int ctx_version;
-    _HPy_real h_None;
-    _HPy_real h_True;
-    _HPy_real h_False;
-    _HPy_real h_ValueError;
+    struct _HPy_s h_None;
+    struct _HPy_s h_True;
+    struct _HPy_s h_False;
+    struct _HPy_s h_ValueError;
     void *ctx_Module_Create;
     void *ctx_Dup;
     void *ctx_Close;
@@ -91,8 +95,6 @@
     HPy_ssize_t m_size;
     HPyMethodDef *m_methods;
 } HPyModuleDef;
-
-
 """)
 
 HPy_ssize_t = cts.gettype('HPy_ssize_t')
@@ -103,9 +105,10 @@
 HPyContext = cts.gettype('HPyContext')
 
 # for practical reason, we use a primitive type to represent HPy almost
-# everywhere in RPython. HOWEVER, the "real" HPy C type is a struct
+# everywhere in RPython: for example, rffi cannot handle functions returning
+# structs. HOWEVER, the "real" HPy C type is a struct, which is available as
+# "_struct_HPy_s"
 HPy = cts.gettype('HPy')
-_HPy_real = cts.gettype('_HPy_real')
 
 HPyInitFunc = cts.gettype('HPyInitFunc')
 _HPyCFunction = cts.gettype('_HPyCFunction')

From pypy.commits at gmail.com  Fri Nov 22 11:26:46 2019
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 22 Nov 2019 08:26:46 -0800 (PST)
Subject: [pypy-commit] pypy hpy-ctypespace: move the last type definition
 into cts
Message-ID: <5dd80c46.1c69fb81.d92ca.3280@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy-ctypespace
Changeset: r98143:3f52349a3cfd
Date: 2019-11-22 17:22 +0100
http://bitbucket.org/pypy/pypy/changeset/3f52349a3cfd/

Log:	move the last type definition into cts

diff --git a/pypy/module/hpy_universal/llapi.py b/pypy/module/hpy_universal/llapi.py
--- a/pypy/module/hpy_universal/llapi.py
+++ b/pypy/module/hpy_universal/llapi.py
@@ -79,7 +79,7 @@
 typedef void *_HPyCPyCFunction; // not used here
 typedef void (*_HPyMethodPairFunc)(_HPyCFunction *out_func,
                                    _HPyCPyCFunction *out_trampoline);
-
+typedef HPy (*HPyMeth_VarArgs)(HPyContext ctx, HPy self, HPy *args, HPy_ssize_t nargs);
 
 typedef struct {
     const char         *ml_name;
@@ -113,9 +113,7 @@
 HPyInitFunc = cts.gettype('HPyInitFunc')
 _HPyCFunction = cts.gettype('_HPyCFunction')
 _HPyCPyCFunction = cts.gettype('_HPyCPyCFunction')
-
-HPyMeth_VarArgs = lltype.Ptr(
-    lltype.FuncType([HPyContext, HPy, lltype.Ptr(rffi.CArray(HPy)), HPy_ssize_t], HPy))
+HPyMeth_VarArgs = cts.gettype('HPyMeth_VarArgs')
 
 HPyMethodDef = cts.gettype('HPyMethodDef')
 HPyModuleDef = cts.gettype('HPyModuleDef')

From pypy.commits at gmail.com  Fri Nov 22 19:09:37 2019
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 22 Nov 2019 16:09:37 -0800 (PST)
Subject: [pypy-commit] pypy hpy: resolve an XXX and use the proper way to
 include src/getargs.c as a separate compiled module
Message-ID: <5dd878c1.1c69fb81.9cb18.318e@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98147:05f41db32719
Date: 2019-11-23 01:08 +0100
http://bitbucket.org/pypy/pypy/changeset/05f41db32719/

Log:	resolve an XXX and use the proper way to include src/getargs.c as a
	separate compiled module

diff --git a/pypy/module/hpy_universal/llapi.py b/pypy/module/hpy_universal/llapi.py
--- a/pypy/module/hpy_universal/llapi.py
+++ b/pypy/module/hpy_universal/llapi.py
@@ -1,6 +1,7 @@
 import py
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
+from rpython.translator import cdir
 from pypy import pypydir
 from pypy.module.hpy_universal import _vendored
 
@@ -8,30 +9,27 @@
 INCLUDE_DIR = PYPYDIR.join('module', 'hpy_universal', '_vendored', 'include')
 SRC_DIR = PYPYDIR.join('module', 'hpy_universal', 'src')
 
-# XXX I don't understand what is going on here: if I put getargs.c as
-# separate_module_files, then ll2ctypes can't find it. I need to #include t in
-# separate_module_sources for now...
-eci = ExternalCompilationInfo(includes=["universal/hpy.h"],
-                              include_dirs=[INCLUDE_DIR],
-                              ## separate_module_files=[
-                              ##     SRC_DIR.join('getargs.c'),
-                              ##     ],
-                              post_include_bits=["""
-RPY_EXTERN void *_HPy_GetGlobalCtx(void);
-RPY_EXTERN int ctx_Arg_Parse(HPyContext ctx, HPy *args, HPy_ssize_t nargs,
-                             const char *fmt, va_list vl);
-"""],
-                              separate_module_sources=["""
+eci = ExternalCompilationInfo(
+    includes=["universal/hpy.h", "getargs.h"],
+    include_dirs=[
+        cdir,        # for precommondefs.h
+        INCLUDE_DIR, # for universal/hpy.h
+        SRC_DIR,     # for getargs.h
+    ],
+    separate_module_files=[
+        SRC_DIR.join('getargs.c'),
+    ],
+    post_include_bits=["""
+        RPY_EXTERN void *_HPy_GetGlobalCtx(void);
+    """],
+    separate_module_sources=["""
+        struct _HPyContext_s hpy_global_ctx;
+        void *_HPy_GetGlobalCtx(void)
+        {
+            return &hpy_global_ctx;
+        }
+    """])
 
-#include "%s"
-
-struct _HPyContext_s hpy_global_ctx;
-void *_HPy_GetGlobalCtx(void)
-{
-    return &hpy_global_ctx;
-}
-
-""" % SRC_DIR.join('getargs.c')])
 
 HPy_ssize_t = lltype.Signed # XXXXXXXXX?
 
diff --git a/pypy/module/hpy_universal/src/getargs.c b/pypy/module/hpy_universal/src/getargs.c
--- a/pypy/module/hpy_universal/src/getargs.c
+++ b/pypy/module/hpy_universal/src/getargs.c
@@ -1,6 +1,5 @@
 #include "universal/hpy.h"
-
-#include <stdio.h>
+#include "getargs.h"
 
 /* XXX: this function is copied&pasted THREE times:
  *     hpy_devel/include/hpy.h
diff --git a/pypy/module/hpy_universal/src/getargs.h b/pypy/module/hpy_universal/src/getargs.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/src/getargs.h
@@ -0,0 +1,6 @@
+#include <stdarg.h>
+#include "universal/hpy.h"
+#include "src/precommondefs.h"
+
+RPY_EXTERN int ctx_Arg_Parse(HPyContext ctx, HPy *args, HPy_ssize_t nargs,
+                             const char *fmt, va_list vl);

From pypy.commits at gmail.com  Fri Nov 22 20:04:34 2019
From: pypy.commits at gmail.com (mattip)
Date: Fri, 22 Nov 2019 17:04:34 -0800 (PST)
Subject: [pypy-commit] pypy cppyy-dev: document branch to be closed
Message-ID: <5dd885a2.1c69fb81.d722b.0297@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cppyy-dev
Changeset: r98149:76fdc207c387
Date: 2019-11-22 17:02 -0800
http://bitbucket.org/pypy/pypy/changeset/76fdc207c387/

Log:	document branch to be closed

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -19,4 +19,15 @@
 
 .. branch: allow-forcing-no-embed
 
-When packaging, allow suppressing embedded dependencies via PYPY_NO_EMBED_DEPENDENCIES
+When packaging, allow suppressing embedded dependencies via
+PYPY_NO_EMBED_DEPENDENCIES
+
+.. branch: int-test-is-zero
+
+.. branch: cppyy-dev
+
+Upgraded the built-in ``_cppyy`` module to ``cppyy-backend 1.10.6``, which
+provides, among others, better template resolution, stricter ``enum`` handling,
+anonymous struct/unions, cmake fragments for distribution, optimizations for
+PODs, and faster wrapper calls.
+

From pypy.commits at gmail.com  Fri Nov 22 20:04:37 2019
From: pypy.commits at gmail.com (mattip)
Date: Fri, 22 Nov 2019 17:04:37 -0800 (PST)
Subject: [pypy-commit] pypy default: remerge cppyy-dev into default which
 updates the backend
Message-ID: <5dd885a5.1c69fb81.bf616.037b@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r98150:490e127a5973
Date: 2019-11-22 17:03 -0800
http://bitbucket.org/pypy/pypy/changeset/490e127a5973/

Log:	remerge cppyy-dev into default which updates the backend

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -19,4 +19,15 @@
 
 .. branch: allow-forcing-no-embed
 
-When packaging, allow suppressing embedded dependencies via PYPY_NO_EMBED_DEPENDENCIES
+When packaging, allow suppressing embedded dependencies via
+PYPY_NO_EMBED_DEPENDENCIES
+
+.. branch: int-test-is-zero
+
+.. branch: cppyy-dev
+
+Upgraded the built-in ``_cppyy`` module to ``cppyy-backend 1.10.6``, which
+provides, among others, better template resolution, stricter ``enum`` handling,
+anonymous struct/unions, cmake fragments for distribution, optimizations for
+PODs, and faster wrapper calls.
+

From pypy.commits at gmail.com  Sat Nov 23 06:06:41 2019
From: pypy.commits at gmail.com (antocuni)
Date: Sat, 23 Nov 2019 03:06:41 -0800 (PST)
Subject: [pypy-commit] pypy hpy-ctypespace: fix these two tests which were
 failing after the change of the repr
Message-ID: <5dd912c1.1c69fb81.e27fb.61ba@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy-ctypespace
Changeset: r98151:be27c10c92d1
Date: 2019-11-23 12:06 +0100
http://bitbucket.org/pypy/pypy/changeset/be27c10c92d1/

Log:	fix these two tests which were failing after the change of the repr

diff --git a/rpython/jit/backend/llsupport/test/test_descr.py b/rpython/jit/backend/llsupport/test/test_descr.py
--- a/rpython/jit/backend/llsupport/test/test_descr.py
+++ b/rpython/jit/backend/llsupport/test/test_descr.py
@@ -426,9 +426,10 @@
     #
     fielddescr = get_field_arraylen_descr(c0, rstr.STR)
     ofs = fielddescr.offset
-    assert repr(ofs) == ("< <FieldOffset <GcStruct rpy_string { hash, chars }>"
-                         " 'chars'> + < ArrayLengthOffset"
-                         " <Array of Char > > >")
+    assert repr(ofs) == (
+        "< <FieldOffset <GcStruct rpy_string { hash, chars }> 'chars'> "
+        "+ < ArrayLengthOffset <Array of Char "
+        "{'extra_item_after_alloc': 1, 'immutable': True} > > >")
     # caching:
     assert fielddescr is get_field_arraylen_descr(c0, rstr.STR)
 
diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py
--- a/rpython/rtyper/lltypesystem/lltype.py
+++ b/rpython/rtyper/lltypesystem/lltype.py
@@ -467,16 +467,16 @@
     _str_fields = saferecursive(_str_fields, '...')
 
     def __str__(self):
-        hints = str(self._hints) if self._hints else ''
-        return "%s of %s %s " % (self.__class__.__name__,
-                                 self._str_fields(),
-                                 hints)
+        hints = (' ' + str(self._hints)) if self._hints else ''
+        return "%s of %s%s " % (self.__class__.__name__,
+                                self._str_fields(),
+                                hints)
 
     def _short_name(self):
-        hints = str(self._hints) if self._hints else ''
-        return "%s %s %s" % (self.__class__.__name__,
-                             self.OF._short_name(),
-                             hints)
+        hints = (' ' + str(self._hints)) if self._hints else ''
+        return "%s %s%s" % (self.__class__.__name__,
+                            self.OF._short_name(),
+                            hints)
     _short_name = saferecursive(_short_name, '...')
 
     def _container_example(self):
diff --git a/rpython/rtyper/lltypesystem/test/test_rffi.py b/rpython/rtyper/lltypesystem/test/test_rffi.py
--- a/rpython/rtyper/lltypesystem/test/test_rffi.py
+++ b/rpython/rtyper/lltypesystem/test/test_rffi.py
@@ -840,7 +840,7 @@
     interpret(test_ptradd, [])
 
 def test_voidptr():
-    assert repr(VOIDP) == "<* Array of void >"
+    assert repr(VOIDP) == "<* Array of void {'nolength': True, 'render_as_void': True} >"
 
 class TestCRffi(BaseTestRffi):
     def compile(self, func, args, **kwds):

From pypy.commits at gmail.com  Sun Nov 24 00:49:14 2019
From: pypy.commits at gmail.com (wlav)
Date: Sat, 23 Nov 2019 21:49:14 -0800 (PST)
Subject: [pypy-commit] pypy default: ensure memory is allocated for
 exception status output
Message-ID: <5dda19da.1c69fb81.5c718.6a04@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: 
Changeset: r98154:adff192a6c41
Date: 2019-11-23 21:48 -0800
http://bitbucket.org/pypy/pypy/changeset/adff192a6c41/

Log:	ensure memory is allocated for exception status output

diff --git a/pypy/module/_cppyy/src/dummy_backend.cxx b/pypy/module/_cppyy/src/dummy_backend.cxx
--- a/pypy/module/_cppyy/src/dummy_backend.cxx
+++ b/pypy/module/_cppyy/src/dummy_backend.cxx
@@ -903,7 +903,8 @@
 
 /* handling of function argument buffer ----------------------------------- */
 void* cppyy_allocate_function_args(int nargs) {
-    CPPYY_G__value* args = (CPPYY_G__value*)malloc(nargs*sizeof(CPPYY_G__value));
+    /* nargs parameters + one unsigned long for exception status output */
+    CPPYY_G__value* args = (CPPYY_G__value*)malloc(nargs*sizeof(CPPYY_G__value)+sizeof(unsigned long));
     for (int i = 0; i < nargs; ++i)
         args[i].type = 'l';
     return (void*)args;

From pypy.commits at gmail.com  Wed Nov 27 02:01:42 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 26 Nov 2019 23:01:42 -0800 (PST)
Subject: [pypy-commit] pypy default: start 7.3.0 release cycle: move
 whatsnew, add release note, change version
Message-ID: <5dde1f56.1c69fb81.64858.8c97@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r98156:dbbbae99135f
Date: 2019-11-26 08:25 -0800
http://bitbucket.org/pypy/pypy/changeset/dbbbae99135f/

Log:	start 7.3.0 release cycle: move whatsnew, add release note, change
	version

diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -73,7 +73,7 @@
 # The short X.Y version.
 version = '7.3'
 # The full version, including alpha/beta/rc tags.
-release = '7.3.0'
+release = '7.3.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-v7.3.0.rst
    release-v7.2.0.rst
    release-v7.1.1.rst
    release-v7.1.0.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-7.3.0.rst
    whatsnew-pypy2-7.2.0.rst
    whatsnew-pypy2-7.1.0.rst
    whatsnew-pypy2-7.0.0.rst
@@ -43,6 +44,7 @@
 
 .. toctree::
     whatsnew-pypy3-head.rst
+    whatsnew-pypy3-7.3.0.rst
     whatsnew-pypy3-7.2.0.rst
     whatsnew-pypy3-7.1.0.rst
 
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-pypy2-7.3.0.rst
rename from pypy/doc/whatsnew-head.rst
rename to pypy/doc/whatsnew-pypy2-7.3.0.rst
diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-7.3.0.rst
rename from pypy/doc/whatsnew-pypy3-head.rst
rename to pypy/doc/whatsnew-pypy3-7.3.0.rst
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -32,8 +32,8 @@
  *     module/sys/version.py
  *     doc/conf.py
  */
-#define PYPY_VERSION "7.3.0-alpha0"
-#define PYPY_VERSION_NUM  0x07030000
+#define PYPY_VERSION "7.3.1-alpha0"
+#define PYPY_VERSION_NUM  0x07030100
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
    stays alive. */
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -13,7 +13,7 @@
 # make sure to keep PYPY_VERSION in sync with:
 #    module/cpyext/include/patchlevel.h
 #    doc/conf.py
-PYPY_VERSION               = (7, 3, 0, "alpha", 0)
+PYPY_VERSION               = (7, 3, 1, "alpha", 0)
 
 
 import pypy

From pypy.commits at gmail.com  Wed Nov 27 02:01:40 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 26 Nov 2019 23:01:40 -0800 (PST)
Subject: [pypy-commit] pypy release-pypy2.7-v7.x: merge default into branch,
 bump version
Message-ID: <5dde1f54.1c69fb81.726e4.b6b7@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-pypy2.7-v7.x
Changeset: r98155:5ec005b3e483
Date: 2019-11-26 06:11 -0800
http://bitbucket.org/pypy/pypy/changeset/5ec005b3e483/

Log:	merge default into branch, bump version

diff too long, truncating to 2000 out of 11835 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -50,4 +50,10 @@
 de061d87e39c7df4e436974096d7982c676a859d release-pypy3.6-v7.1.0
 784b254d669919c872a505b807db8462b6140973 release-pypy3.6-v7.1.1
 8cdda8b8cdb8ff29d9e620cccd6c5edd2f2a23ec release-pypy2.7-v7.1.1
-
+85dae4fd5c234b482feff834c73e089872194541 release-pypy2.7-v7.2.0rc0
+7ffb92269488f37c707ce66076f50ffd8613f8e2 release-pypy3.6-v7.2.0rc0
+4d6761df14ffd6f38450f183ac1fad32c946c21b release-pypy3.6-v7.2.0rc1
+5da45ced70e515f94686be0df47c59abd1348ebc release-pypy3.6-v7.2.0rc2
+4a68d8d3d2fc1faec2e83bcb4d28559099092574 release-pypy2.7-v7.2.0rc2
+4a68d8d3d2fc1faec2e83bcb4d28559099092574 release-pypy2.7-v7.2.0
+5da45ced70e515f94686be0df47c59abd1348ebc release-pypy3.6-v7.2.0
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -3,10 +3,11 @@
 License
 =======
 
-Except when otherwise stated (look for LICENSE files in directories or
-information at the beginning of each file) all software and documentation in
-the 'rpython', 'pypy', 'ctype_configure', 'dotviewer', 'demo', 'lib_pypy',
-'py', and '_pytest' directories is licensed as follows:
+Except when otherwise stated (look for LICENSE files in directories
+or information at the beginning of each file) all software and
+documentation in the 'rpython', 'pypy', 'ctype_configure', 'dotviewer',
+'demo', 'extra_tests', 'include', 'lib_pypy', 'py', and '_pytest'
+directories is licensed as follows:
 
     The MIT License
 
diff --git a/README.rst b/README.rst
--- a/README.rst
+++ b/README.rst
@@ -9,15 +9,15 @@
 
 The home page for the interpreter is:
 
-    http://pypy.org/
+    https://pypy.org/
 
 If you want to help developing PyPy, this documentation might help you:
 
-    http://doc.pypy.org/
+    https://doc.pypy.org/
 
 More documentation about the RPython framework can be found here:
 
-    http://rpython.readthedocs.io/
+    https://rpython.readthedocs.io/
 
 The source for the documentation is in the pypy/doc directory.
 
@@ -25,7 +25,7 @@
 Using PyPy instead of CPython
 -----------------------------
 
-Please read the information at http://pypy.org/ to find the correct way to
+Please read the information at https://pypy.org/ to find the correct way to
 download and use PyPy as an alternative to CPython. 
 
 
@@ -36,7 +36,7 @@
 interpreter. It is time-consuming and requires significant computing resources.
 More information can be found here:
 
-    http://doc.pypy.org/en/latest/build.html
+    https://doc.pypy.org/en/latest/build.html
 
 Enjoy and send us feedback!
 
diff --git a/extra_tests/cffi_tests/cffi0/test_function.py b/extra_tests/cffi_tests/cffi0/test_function.py
--- a/extra_tests/cffi_tests/cffi0/test_function.py
+++ b/extra_tests/cffi_tests/cffi0/test_function.py
@@ -114,7 +114,7 @@
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
             int fputs(const char *, void *);
-            void *stderr;
+            extern void *stderr;
         """)
         needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
@@ -131,7 +131,7 @@
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
             int fputs(char *, void *);
-            void *stderr;
+            extern void *stderr;
         """)
         needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
@@ -148,7 +148,7 @@
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
            int fprintf(void *, const char *format, ...);
-           void *stderr;
+           extern void *stderr;
         """)
         needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
@@ -210,7 +210,7 @@
             py.test.skip("probably no symbol 'stderr' in the lib")
         ffi.cdef("""
             int fputs(const char *, void *);
-            void *stderr;
+            extern void *stderr;
         """)
         needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
@@ -257,7 +257,7 @@
             py.test.skip("probably no symbol 'stdout' in the lib")
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
-            void *stdout;
+            extern void *stdout;
         """)
         needs_dlopen_none()
         C = ffi.dlopen(None)
@@ -497,7 +497,7 @@
         ffi.cdef("""
             typedef enum { MYE1, MYE2 } myenum_t;
             double myfunc(double);
-            double myvar;
+            extern double myvar;
             const double myconst;
             #define MYFOO 42
         """)
@@ -508,7 +508,7 @@
         if self.Backend is CTypesBackend:
             py.test.skip("not with the ctypes backend")
         ffi = FFI(backend=self.Backend())
-        ffi.cdef("int foobar(void); int foobaz;")
+        ffi.cdef("int foobar(void); extern int foobaz;")
         lib = ffi.dlopen(lib_m)
         ffi.dlclose(lib)
         e = py.test.raises(ValueError, getattr, lib, 'foobar')
diff --git a/extra_tests/cffi_tests/cffi0/test_ownlib.py b/extra_tests/cffi_tests/cffi0/test_ownlib.py
--- a/extra_tests/cffi_tests/cffi0/test_ownlib.py
+++ b/extra_tests/cffi_tests/cffi0/test_ownlib.py
@@ -202,7 +202,7 @@
             py.test.skip("fix the auto-generation of the tiny test lib")
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
-            int my_array[7];
+            extern int my_array[7];
         """)
         ownlib = ffi.dlopen(self.module)
         for i in range(7):
@@ -224,7 +224,7 @@
             py.test.skip("not supported by the ctypes backend")
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
-            int my_array[];
+            extern int my_array[];
         """)
         ownlib = ffi.dlopen(self.module)
         for i in range(7):
@@ -292,7 +292,7 @@
                 long bottom;
             } RECT;
             
-            long left, top, right, bottom;
+            extern long left, top, right, bottom;
 
             RECT ReturnRect(int i, RECT ar, RECT* br, POINT cp, RECT dr,
                         RECT *er, POINT fp, RECT gr);
@@ -322,7 +322,7 @@
         if self.Backend is CTypesBackend:
             py.test.skip("not implemented with the ctypes backend")
         ffi = FFI(backend=self.Backend())
-        ffi.cdef("long left; int test_getting_errno(void);")
+        ffi.cdef("extern long left; int test_getting_errno(void);")
         lib = ffi.dlopen(self.module)
         lib.left = 123456
         p = ffi.addressof(lib, "left")
diff --git a/extra_tests/cffi_tests/cffi0/test_parsing.py b/extra_tests/cffi_tests/cffi0/test_parsing.py
--- a/extra_tests/cffi_tests/cffi0/test_parsing.py
+++ b/extra_tests/cffi_tests/cffi0/test_parsing.py
@@ -325,6 +325,7 @@
     assert value == sys.maxsize * 2 - 40
 
 def test__is_constant_globalvar():
+    import warnings
     for input, expected_output in [
         ("int a;",          False),
         ("const int a;",    True),
@@ -342,10 +343,13 @@
         ("const int a[5][6];", False),
         ]:
         ffi = FFI()
-        ffi.cdef(input)
+        with warnings.catch_warnings(record=True) as log:
+            warnings.simplefilter("always")
+            ffi.cdef(input)
         declarations = ffi._parser._declarations
         assert ('constant a' in declarations) == expected_output
         assert ('variable a' in declarations) == (not expected_output)
+        assert len(log) == (1 - expected_output)
 
 def test_restrict():
     from cffi import model
@@ -355,7 +359,7 @@
         ("int *a;",            False),
         ]:
         ffi = FFI()
-        ffi.cdef(input)
+        ffi.cdef("extern " + input)
         tp, quals = ffi._parser._declarations['variable a']
         assert bool(quals & model.Q_RESTRICT) == expected_output
 
diff --git a/extra_tests/cffi_tests/cffi0/test_verify.py b/extra_tests/cffi_tests/cffi0/test_verify.py
--- a/extra_tests/cffi_tests/cffi0/test_verify.py
+++ b/extra_tests/cffi_tests/cffi0/test_verify.py
@@ -4,6 +4,7 @@
 import sys, os, math, weakref
 from cffi import FFI, VerificationError, VerificationMissing, model, FFIError
 from extra_tests.cffi_tests.support import *
+from extra_tests.cffi_tests.support import extra_compile_args
 
 
 lib_m = ['m']
@@ -14,17 +15,6 @@
         lib_m = ['msvcrt']
     pass      # no obvious -Werror equivalent on MSVC
 else:
-    if (sys.platform == 'darwin' and
-          [int(x) for x in os.uname()[2].split('.')] >= [11, 0, 0]):
-        # assume a standard clang or gcc
-        extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion']
-        # special things for clang
-        extra_compile_args.append('-Qunused-arguments')
-    else:
-        # assume a standard gcc
-        extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion',
-                              '-Wno-unused-parameter']
-
     class FFI(FFI):
         def verify(self, *args, **kwds):
             return super(FFI, self).verify(
@@ -287,7 +277,7 @@
 def test_var_signed_integer_types():
     ffi = FFI()
     lst = all_signed_integer_types(ffi)
-    csource = "\n".join(["%s somevar_%s;" % (tp, tp.replace(' ', '_'))
+    csource = "\n".join(["static %s somevar_%s;" % (tp, tp.replace(' ', '_'))
                          for tp in lst])
     ffi.cdef(csource)
     lib = ffi.verify(csource)
@@ -306,7 +296,7 @@
 def test_var_unsigned_integer_types():
     ffi = FFI()
     lst = all_unsigned_integer_types(ffi)
-    csource = "\n".join(["%s somevar_%s;" % (tp, tp.replace(' ', '_'))
+    csource = "\n".join(["static %s somevar_%s;" % (tp, tp.replace(' ', '_'))
                          for tp in lst])
     ffi.cdef(csource)
     lib = ffi.verify(csource)
@@ -818,8 +808,8 @@
 
 def test_access_variable():
     ffi = FFI()
-    ffi.cdef("int foo(void);\n"
-             "int somenumber;")
+    ffi.cdef("static int foo(void);\n"
+             "static int somenumber;")
     lib = ffi.verify("""
         static int somenumber = 2;
         static int foo(void) {
@@ -836,7 +826,7 @@
 def test_access_address_of_variable():
     # access the address of 'somenumber': need a trick
     ffi = FFI()
-    ffi.cdef("int somenumber; static int *const somenumberptr;")
+    ffi.cdef("static int somenumber; static int *const somenumberptr;")
     lib = ffi.verify("""
         static int somenumber = 2;
         #define somenumberptr (&somenumber)
@@ -849,7 +839,7 @@
 def test_access_array_variable(length=5):
     ffi = FFI()
     ffi.cdef("int foo(int);\n"
-             "int somenumber[%s];" % (length,))
+             "static int somenumber[%s];" % (length,))
     lib = ffi.verify("""
         static int somenumber[] = {2, 2, 3, 4, 5};
         static int foo(int i) {
@@ -881,7 +871,7 @@
     ffi = FFI()
     ffi.cdef("struct foo { int x; ...; };\n"
              "int foo(int);\n"
-             "struct foo stuff;")
+             "static struct foo stuff;")
     lib = ffi.verify("""
         struct foo { int x, y, z; };
         static struct foo stuff = {2, 5, 8};
@@ -905,9 +895,9 @@
 
 def test_access_callback():
     ffi = FFI()
-    ffi.cdef("int (*cb)(int);\n"
-             "int foo(int);\n"
-             "void reset_cb(void);")
+    ffi.cdef("static int (*cb)(int);\n"
+             "static int foo(int);\n"
+             "static void reset_cb(void);")
     lib = ffi.verify("""
         static int g(int x) { return x * 7; }
         static int (*cb)(int);
@@ -923,9 +913,9 @@
 def test_access_callback_function_typedef():
     ffi = FFI()
     ffi.cdef("typedef int mycallback_t(int);\n"
-             "mycallback_t *cb;\n"
-             "int foo(int);\n"
-             "void reset_cb(void);")
+             "static mycallback_t *cb;\n"
+             "static int foo(int);\n"
+             "static void reset_cb(void);")
     lib = ffi.verify("""
         static int g(int x) { return x * 7; }
         static int (*cb)(int);
@@ -1075,7 +1065,7 @@
 def test_autofilled_struct_as_argument_dynamic():
     ffi = FFI()
     ffi.cdef("struct foo_s { long a; ...; };\n"
-             "int (*foo)(struct foo_s);")
+             "static int (*foo)(struct foo_s);")
     lib = ffi.verify("""
         struct foo_s {
             double b;
@@ -1084,7 +1074,7 @@
         int foo1(struct foo_s s) {
             return (int)s.a - (int)s.b;
         }
-        int (*foo)(struct foo_s s) = &foo1;
+        static int (*foo)(struct foo_s s) = &foo1;
     """)
     e = py.test.raises(NotImplementedError, lib.foo, "?")
     msg = ("ctype 'struct foo_s' not supported as argument.  It is a struct "
@@ -1454,7 +1444,7 @@
         py.test.skip("_Bool not in MSVC")
     ffi = FFI()
     ffi.cdef("struct foo_s { _Bool x; };"
-             "_Bool foo(_Bool); _Bool (*foop)(_Bool);")
+             "_Bool foo(_Bool); static _Bool (*foop)(_Bool);")
     lib = ffi.verify("""
         struct foo_s { _Bool x; };
         int foo(int arg) {
@@ -1463,7 +1453,7 @@
         _Bool _foofunc(_Bool x) {
             return !x;
         }
-        _Bool (*foop)(_Bool) = _foofunc;
+        static _Bool (*foop)(_Bool) = _foofunc;
     """)
     p = ffi.new("struct foo_s *")
     p.x = 1
@@ -1654,7 +1644,7 @@
 
 def test_FILE_stored_explicitly():
     ffi = FFI()
-    ffi.cdef("int myprintf11(const char *, int); FILE *myfile;")
+    ffi.cdef("int myprintf11(const char *, int); extern FILE *myfile;")
     lib = ffi.verify("""
         #include <stdio.h>
         FILE *myfile;
@@ -1680,19 +1670,19 @@
 
 def test_global_array_with_missing_length():
     ffi = FFI()
-    ffi.cdef("int fooarray[];")
+    ffi.cdef("extern int fooarray[];")
     lib = ffi.verify("int fooarray[50];")
     assert repr(lib.fooarray).startswith("<cdata 'int *'")
 
 def test_global_array_with_dotdotdot_length():
     ffi = FFI()
-    ffi.cdef("int fooarray[...];")
+    ffi.cdef("extern int fooarray[...];")
     lib = ffi.verify("int fooarray[50];")
     assert repr(lib.fooarray).startswith("<cdata 'int[50]'")
 
 def test_bad_global_array_with_dotdotdot_length():
     ffi = FFI()
-    ffi.cdef("int fooarray[...];")
+    ffi.cdef("extern int fooarray[...];")
     py.test.raises(VerificationError, ffi.verify, "char fooarray[23];")
 
 def test_struct_containing_struct():
@@ -1813,7 +1803,7 @@
 def test_callback_indirection():
     ffi = FFI()
     ffi.cdef("""
-        int (*python_callback)(int how_many, int *values);
+        static int (*python_callback)(int how_many, int *values);
         int (*const c_callback)(int,...);   /* pass this ptr to C routines */
         int some_c_function(int(*cb)(int,...));
     """)
@@ -1947,24 +1937,24 @@
 
 def test_bug_const_char_ptr_array_1():
     ffi = FFI()
-    ffi.cdef("""const char *a[...];""")
+    ffi.cdef("""extern const char *a[...];""")
     lib = ffi.verify("""const char *a[5];""")
     assert repr(ffi.typeof(lib.a)) == "<ctype 'char *[5]'>"
 
 def test_bug_const_char_ptr_array_2():
     from cffi import FFI     # ignore warnings
     ffi = FFI()
-    ffi.cdef("""const int a[];""")
+    ffi.cdef("""extern const int a[];""")
     lib = ffi.verify("""const int a[5];""")
     assert repr(ffi.typeof(lib.a)) == "<ctype 'int *'>"
 
 def _test_various_calls(force_libffi):
     cdef_source = """
-    int xvalue;
-    long long ivalue, rvalue;
-    float fvalue;
-    double dvalue;
-    long double Dvalue;
+    extern int xvalue;
+    extern long long ivalue, rvalue;
+    extern float fvalue;
+    extern double dvalue;
+    extern long double Dvalue;
     signed char tf_bb(signed char x, signed char c);
     unsigned char tf_bB(signed char x, unsigned char c);
     short tf_bh(signed char x, short c);
@@ -2148,7 +2138,7 @@
     # exported symbols as well.  So we must not export a simple name
     # like 'foo'!
     ffi1 = FFI()
-    ffi1.cdef("int foo_verify_dlopen_flags;")
+    ffi1.cdef("extern int foo_verify_dlopen_flags;")
 
     lib1 = ffi1.verify("int foo_verify_dlopen_flags;",
                        flags=ffi1.RTLD_GLOBAL | ffi1.RTLD_LAZY)
@@ -2162,7 +2152,7 @@
 def get_second_lib():
     # Hack, using modulename makes the test fail
     ffi2 = FFI()
-    ffi2.cdef("int foo_verify_dlopen_flags;")
+    ffi2.cdef("extern int foo_verify_dlopen_flags;")
     lib2 = ffi2.verify("int foo_verify_dlopen_flags;",
                        flags=ffi2.RTLD_GLOBAL | ffi2.RTLD_LAZY)
     return lib2
diff --git a/extra_tests/cffi_tests/cffi1/test_dlopen.py b/extra_tests/cffi_tests/cffi1/test_dlopen.py
--- a/extra_tests/cffi_tests/cffi1/test_dlopen.py
+++ b/extra_tests/cffi_tests/cffi1/test_dlopen.py
@@ -7,7 +7,7 @@
 
 def test_simple():
     ffi = FFI()
-    ffi.cdef("int close(int); static const int BB = 42; int somevar;")
+    ffi.cdef("int close(int); static const int BB = 42; extern int somevar;")
     target = udir.join('test_simple.py')
     make_py_source(ffi, 'test_simple', str(target))
     assert target.read() == r"""# auto-generated file
@@ -197,7 +197,7 @@
 
 def test_global_var():
     ffi = FFI()
-    ffi.cdef("int myglob;")
+    ffi.cdef("extern int myglob;")
     target = udir.join('test_global_var.py')
     make_py_source(ffi, 'test_global_var', str(target))
     assert target.read() == r"""# auto-generated file
diff --git a/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py
--- a/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py
+++ b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py
@@ -1780,7 +1780,7 @@
 
     def test_import_from_lib(self):
         ffi2 = cffi.FFI()
-        ffi2.cdef("int myfunc(int); int myvar;\n#define MYFOO ...\n")
+        ffi2.cdef("int myfunc(int); extern int myvar;\n#define MYFOO ...\n")
         outputfilename = recompile(ffi2, "_test_import_from_lib",
                                    "int myfunc(int x) { return x + 1; }\n"
                                    "int myvar = -5;\n"
diff --git a/extra_tests/cffi_tests/cffi1/test_re_python.py b/extra_tests/cffi_tests/cffi1/test_re_python.py
--- a/extra_tests/cffi_tests/cffi1/test_re_python.py
+++ b/extra_tests/cffi_tests/cffi1/test_re_python.py
@@ -64,11 +64,11 @@
     #define BIGNEG -420000000000L
     int add42(int);
     int add43(int, ...);
-    int globalvar42;
+    extern int globalvar42;
     const int globalconst42;
     const char *const globalconsthello;
     int no_such_function(int);
-    int no_such_globalvar;
+    extern int no_such_globalvar;
     struct foo_s;
     typedef struct bar_s { int x; signed char a[]; } bar_t;
     enum foo_e { AA, BB, CC };
@@ -76,6 +76,7 @@
     struct with_union { union { int a; char b; }; };
     union with_struct { struct { int a; char b; }; };
     struct NVGcolor { union { float rgba[4]; struct { float r,g,b,a; }; }; };
+    typedef struct selfref { struct selfref *next; } *selfref_ptr_t;
     """)
     ffi.set_source('re_python_pysrc', None)
     ffi.emit_python_code(str(tmpdir.join('re_python_pysrc.py')))
@@ -255,3 +256,8 @@
     assert ffi.offsetof("struct NVGcolor", "g") == FLOAT
     assert ffi.offsetof("struct NVGcolor", "b") == FLOAT * 2
     assert ffi.offsetof("struct NVGcolor", "a") == FLOAT * 3
+
+def test_selfref():
+    # based on issue #429
+    from re_python_pysrc import ffi
+    ffi.new("selfref_ptr_t")
diff --git a/extra_tests/cffi_tests/cffi1/test_recompiler.py b/extra_tests/cffi_tests/cffi1/test_recompiler.py
--- a/extra_tests/cffi_tests/cffi1/test_recompiler.py
+++ b/extra_tests/cffi_tests/cffi1/test_recompiler.py
@@ -35,8 +35,9 @@
         source = 'extern "C" {\n%s\n}' % (source,)
     elif sys.platform != 'win32':
         # add '-Werror' to the existing 'extra_compile_args' flags
+        from extra_tests.cffi_tests.support import extra_compile_args
         kwds['extra_compile_args'] = (kwds.get('extra_compile_args', []) +
-                                      ['-Werror'])
+                                      extra_compile_args)
     return _verify(ffi, module_name, source, *args, **kwds)
 
 def test_set_source_no_slashes():
@@ -84,7 +85,7 @@
                      "(FUNCTION 1)(PRIMITIVE 7)(FUNCTION_END 1)(POINTER 0)")
 
 def test_type_table_array():
-    check_type_table("int a[100];",
+    check_type_table("extern int a[100];",
                      "(PRIMITIVE 7)(ARRAY 0)(None 100)")
 
 def test_type_table_typedef():
@@ -159,7 +160,7 @@
 
 def test_global_var_array():
     ffi = FFI()
-    ffi.cdef("int a[100];")
+    ffi.cdef("extern int a[100];")
     lib = verify(ffi, 'test_global_var_array', 'int a[100] = { 9999 };')
     lib.a[42] = 123456
     assert lib.a[42] == 123456
@@ -183,7 +184,7 @@
 
 def test_global_var_int():
     ffi = FFI()
-    ffi.cdef("int a, b, c;")
+    ffi.cdef("extern int a, b, c;")
     lib = verify(ffi, 'test_global_var_int', 'int a = 999, b, c;')
     assert lib.a == 999
     lib.a -= 1001
@@ -284,7 +285,7 @@
 
 def test_dir():
     ffi = FFI()
-    ffi.cdef("int ff(int); int aa; static const int my_constant;")
+    ffi.cdef("int ff(int); extern int aa; static const int my_constant;")
     lib = verify(ffi, 'test_dir', """
         #define my_constant  (-45)
         int aa;
@@ -406,7 +407,7 @@
 
 def test_dotdotdot_global_array():
     ffi = FFI()
-    ffi.cdef("int aa[...]; int bb[...];")
+    ffi.cdef("extern int aa[...]; extern int bb[...];")
     lib = verify(ffi, 'test_dotdotdot_global_array',
                  "int aa[41]; int bb[12];")
     assert ffi.sizeof(lib.aa) == 41 * 4
@@ -561,37 +562,37 @@
 
 def test_bad_size_of_global_1():
     ffi = FFI()
-    ffi.cdef("short glob;")
+    ffi.cdef("extern short glob;")
     py.test.raises(VerificationError, verify, ffi,
                    "test_bad_size_of_global_1", "long glob;")
 
 def test_bad_size_of_global_2():
     ffi = FFI()
-    ffi.cdef("int glob[10];")
+    ffi.cdef("extern int glob[10];")
     py.test.raises(VerificationError, verify, ffi,
                    "test_bad_size_of_global_2", "int glob[9];")
 
 def test_unspecified_size_of_global_1():
     ffi = FFI()
-    ffi.cdef("int glob[];")
+    ffi.cdef("extern int glob[];")
     lib = verify(ffi, "test_unspecified_size_of_global_1", "int glob[10];")
     assert ffi.typeof(lib.glob) == ffi.typeof("int *")
 
 def test_unspecified_size_of_global_2():
     ffi = FFI()
-    ffi.cdef("int glob[][5];")
+    ffi.cdef("extern int glob[][5];")
     lib = verify(ffi, "test_unspecified_size_of_global_2", "int glob[10][5];")
     assert ffi.typeof(lib.glob) == ffi.typeof("int(*)[5]")
 
 def test_unspecified_size_of_global_3():
     ffi = FFI()
-    ffi.cdef("int glob[][...];")
+    ffi.cdef("extern int glob[][...];")
     lib = verify(ffi, "test_unspecified_size_of_global_3", "int glob[10][5];")
     assert ffi.typeof(lib.glob) == ffi.typeof("int(*)[5]")
 
 def test_unspecified_size_of_global_4():
     ffi = FFI()
-    ffi.cdef("int glob[...][...];")
+    ffi.cdef("extern int glob[...][...];")
     lib = verify(ffi, "test_unspecified_size_of_global_4", "int glob[10][5];")
     assert ffi.typeof(lib.glob) == ffi.typeof("int[10][5]")
 
@@ -814,7 +815,7 @@
 def test_address_of_global_var():
     ffi = FFI()
     ffi.cdef("""
-        long bottom, bottoms[2];
+        extern long bottom, bottoms[2];
         long FetchRectBottom(void);
         long FetchRectBottoms1(void);
         #define FOOBAR 42
@@ -969,7 +970,7 @@
     ffi = FFI()
     ffi.cdef("""
         typedef ... opaque_t;
-        opaque_t globvar;
+        extern opaque_t globvar;
     """)
     lib = verify(ffi, 'test_variable_of_unknown_size', """
         typedef char opaque_t[6];
@@ -1014,7 +1015,7 @@
 def test_call_with_incomplete_structs():
     ffi = FFI()
     ffi.cdef("typedef struct {...;} foo_t; "
-             "foo_t myglob; "
+             "extern foo_t myglob; "
              "foo_t increment(foo_t s); "
              "double getx(foo_t s);")
     lib = verify(ffi, 'test_call_with_incomplete_structs', """
@@ -1058,7 +1059,7 @@
 
 def test_global_var_array_2():
     ffi = FFI()
-    ffi.cdef("int a[...][...];")
+    ffi.cdef("extern int a[...][...];")
     lib = verify(ffi, 'test_global_var_array_2', 'int a[10][8];')
     lib.a[9][7] = 123456
     assert lib.a[9][7] == 123456
@@ -1071,7 +1072,7 @@
 
 def test_global_var_array_3():
     ffi = FFI()
-    ffi.cdef("int a[][...];")
+    ffi.cdef("extern int a[][...];")
     lib = verify(ffi, 'test_global_var_array_3', 'int a[10][8];')
     lib.a[9][7] = 123456
     assert lib.a[9][7] == 123456
@@ -1082,7 +1083,7 @@
 
 def test_global_var_array_4():
     ffi = FFI()
-    ffi.cdef("int a[10][...];")
+    ffi.cdef("extern int a[10][...];")
     lib = verify(ffi, 'test_global_var_array_4', 'int a[10][8];')
     lib.a[9][7] = 123456
     assert lib.a[9][7] == 123456
@@ -1205,7 +1206,7 @@
 
 def test_import_from_lib():
     ffi = FFI()
-    ffi.cdef("int mybar(int); int myvar;\n#define MYFOO ...")
+    ffi.cdef("int mybar(int); static int myvar;\n#define MYFOO ...")
     lib = verify(ffi, 'test_import_from_lib',
                  "#define MYFOO 42\n"
                  "static int mybar(int x) { return x + 1; }\n"
@@ -1221,7 +1222,7 @@
 
 def test_macro_var_callback():
     ffi = FFI()
-    ffi.cdef("int my_value; int *(*get_my_value)(void);")
+    ffi.cdef("extern int my_value; extern int *(*get_my_value)(void);")
     lib = verify(ffi, 'test_macro_var_callback',
                  "int *(*get_my_value)(void);\n"
                  "#define my_value (*get_my_value())")
@@ -1336,7 +1337,7 @@
 
 def test_const_function_type_args():
     ffi = FFI()
-    ffi.cdef("""int (*foobar)(const int a, const int *b, const int c[]);""")
+    ffi.cdef("""extern int(*foobar)(const int a,const int*b,const int c[]);""")
     lib = verify(ffi, 'test_const_function_type_args', """
         int (*foobar)(const int a, const int *b, const int c[]);
     """)
@@ -1626,7 +1627,7 @@
 
 def test_extern_python_bogus_name():
     ffi = FFI()
-    ffi.cdef("int abc;")
+    ffi.cdef("extern int abc;")
     lib = verify(ffi, 'test_extern_python_bogus_name', "int abc;")
     def fn():
         pass
@@ -1787,8 +1788,8 @@
     ffi.cdef("""
         extern "Python" int __stdcall foo(int);
         extern "Python" int WINAPI bar(int);
-        int (__stdcall * mycb1)(int);
-        int indirect_call(int);
+        static int (__stdcall * mycb1)(int);
+        static int indirect_call(int);
     """)
     lib = verify(ffi, 'test_extern_python_stdcall', """
         #ifndef _MSC_VER
@@ -1856,7 +1857,7 @@
 
 def test_introspect_global_var():
     ffi = FFI()
-    ffi.cdef("float g1;")
+    ffi.cdef("extern float g1;")
     lib = verify(ffi, 'test_introspect_global_var', """
         float g1;
     """)
@@ -1867,7 +1868,7 @@
 
 def test_introspect_global_var_array():
     ffi = FFI()
-    ffi.cdef("float g1[100];")
+    ffi.cdef("extern float g1[100];")
     lib = verify(ffi, 'test_introspect_global_var_array', """
         float g1[100];
     """)
@@ -2039,7 +2040,7 @@
     ffi.cdef("float _Complex f1(float a, float b);");
     lib = verify(ffi, "test_function_returns_float_complex", """
         #include <complex.h>
-        static float _Complex f1(float a, float b) { return a + I*2.0*b; }
+        static float _Complex f1(float a, float b) { return a + I*2.0f*b; }
     """, no_cpp=True)    # <complex.h> fails on some systems with C++
     result = lib.f1(1.25, 5.1)
     assert type(result) == complex
@@ -2090,7 +2091,7 @@
     ffi = FFI()
     ffi.cdef("""
         typedef int foo_t[...], bar_t[...];
-        int gv[...];
+        extern int gv[...];
         typedef int mat_t[...][...];
         typedef int vmat_t[][...];
         """)
diff --git a/extra_tests/cffi_tests/cffi1/test_verify1.py b/extra_tests/cffi_tests/cffi1/test_verify1.py
--- a/extra_tests/cffi_tests/cffi1/test_verify1.py
+++ b/extra_tests/cffi_tests/cffi1/test_verify1.py
@@ -5,7 +5,7 @@
 from cffi import CDefError
 from cffi import recompiler
 from extra_tests.cffi_tests.support import *
-from extra_tests.cffi_tests.support import _verify
+from extra_tests.cffi_tests.support import _verify, extra_compile_args
 import _cffi_backend
 
 lib_m = ['m']
@@ -14,18 +14,6 @@
     import distutils.ccompiler
     if distutils.ccompiler.get_default_compiler() == 'msvc':
         lib_m = ['msvcrt']
-    extra_compile_args = []      # no obvious -Werror equivalent on MSVC
-else:
-    if (sys.platform == 'darwin' and
-          [int(x) for x in os.uname()[2].split('.')] >= [11, 0, 0]):
-        # assume a standard clang or gcc
-        extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion']
-        # special things for clang
-        extra_compile_args.append('-Qunused-arguments')
-    else:
-        # assume a standard gcc
-        extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion',
-                              '-Wno-unused-parameter']
 
 class FFI(FFI):
     error = _cffi_backend.FFI.error
@@ -268,7 +256,7 @@
 def test_var_signed_integer_types():
     ffi = FFI()
     lst = all_signed_integer_types(ffi)
-    csource = "\n".join(["%s somevar_%s;" % (tp, tp.replace(' ', '_'))
+    csource = "\n".join(["static %s somevar_%s;" % (tp, tp.replace(' ', '_'))
                          for tp in lst])
     ffi.cdef(csource)
     lib = ffi.verify(csource)
@@ -287,7 +275,7 @@
 def test_var_unsigned_integer_types():
     ffi = FFI()
     lst = all_unsigned_integer_types(ffi)
-    csource = "\n".join(["%s somevar_%s;" % (tp, tp.replace(' ', '_'))
+    csource = "\n".join(["static %s somevar_%s;" % (tp, tp.replace(' ', '_'))
                          for tp in lst])
     ffi.cdef(csource)
     lib = ffi.verify(csource)
@@ -791,8 +779,8 @@
 
 def test_access_variable():
     ffi = FFI()
-    ffi.cdef("int foo(void);\n"
-             "int somenumber;")
+    ffi.cdef("static int foo(void);\n"
+             "static int somenumber;")
     lib = ffi.verify("""
         static int somenumber = 2;
         static int foo(void) {
@@ -809,7 +797,7 @@
 def test_access_address_of_variable():
     # access the address of 'somenumber': need a trick
     ffi = FFI()
-    ffi.cdef("int somenumber; static int *const somenumberptr;")
+    ffi.cdef("static int somenumber; static int *const somenumberptr;")
     lib = ffi.verify("""
         static int somenumber = 2;
         #define somenumberptr (&somenumber)
@@ -821,8 +809,8 @@
 
 def test_access_array_variable(length=5):
     ffi = FFI()
-    ffi.cdef("int foo(int);\n"
-             "int somenumber[%s];" % (length,))
+    ffi.cdef("static int foo(int);\n"
+             "static int somenumber[%s];" % (length,))
     lib = ffi.verify("""
         static int somenumber[] = {2, 2, 3, 4, 5};
         static int foo(int i) {
@@ -853,8 +841,8 @@
 def test_access_struct_variable():
     ffi = FFI()
     ffi.cdef("struct foo { int x; ...; };\n"
-             "int foo(int);\n"
-             "struct foo stuff;")
+             "static int foo(int);\n"
+             "static struct foo stuff;")
     lib = ffi.verify("""
         struct foo { int x, y, z; };
         static struct foo stuff = {2, 5, 8};
@@ -878,9 +866,9 @@
 
 def test_access_callback():
     ffi = FFI()
-    ffi.cdef("int (*cb)(int);\n"
-             "int foo(int);\n"
-             "void reset_cb(void);")
+    ffi.cdef("static int (*cb)(int);\n"
+             "static int foo(int);\n"
+             "static void reset_cb(void);")
     lib = ffi.verify("""
         static int g(int x) { return x * 7; }
         static int (*cb)(int);
@@ -896,9 +884,9 @@
 def test_access_callback_function_typedef():
     ffi = FFI()
     ffi.cdef("typedef int mycallback_t(int);\n"
-             "mycallback_t *cb;\n"
-             "int foo(int);\n"
-             "void reset_cb(void);")
+             "static mycallback_t *cb;\n"
+             "static int foo(int);\n"
+             "static void reset_cb(void);")
     lib = ffi.verify("""
         static int g(int x) { return x * 7; }
         static int (*cb)(int);
@@ -1039,7 +1027,7 @@
 def test_autofilled_struct_as_argument_dynamic():
     ffi = FFI()
     ffi.cdef("struct foo_s { long a; ...; };\n"
-             "int (*foo)(struct foo_s);")
+             "static int (*foo)(struct foo_s);")
     lib = ffi.verify("""
         struct foo_s {
             double b;
@@ -1048,7 +1036,7 @@
         int foo1(struct foo_s s) {
             return (int)s.a - (int)s.b;
         }
-        int (*foo)(struct foo_s s) = &foo1;
+        static int (*foo)(struct foo_s s) = &foo1;
     """)
     e = py.test.raises(NotImplementedError, lib.foo, "?")
     msg = ("ctype 'struct foo_s' not supported as argument.  It is a struct "
@@ -1424,7 +1412,7 @@
         py.test.skip("_Bool not in MSVC")
     ffi = FFI()
     ffi.cdef("struct foo_s { _Bool x; };"
-             "_Bool foo(_Bool); _Bool (*foop)(_Bool);")
+             "_Bool foo(_Bool); static _Bool (*foop)(_Bool);")
     lib = ffi.verify("""
         struct foo_s { _Bool x; };
         int foo(int arg) {
@@ -1433,7 +1421,7 @@
         _Bool _foofunc(_Bool x) {
             return !x;
         }
-        _Bool (*foop)(_Bool) = _foofunc;
+        static _Bool (*foop)(_Bool) = _foofunc;
     """)
     p = ffi.new("struct foo_s *")
     p.x = 1
@@ -1618,7 +1606,7 @@
 
 def test_FILE_stored_explicitly():
     ffi = FFI()
-    ffi.cdef("int myprintf11(const char *, int); FILE *myfile;")
+    ffi.cdef("int myprintf11(const char *, int); extern FILE *myfile;")
     lib = ffi.verify("""
         #include <stdio.h>
         FILE *myfile;
@@ -1644,13 +1632,13 @@
 
 def test_global_array_with_missing_length():
     ffi = FFI()
-    ffi.cdef("int fooarray[];")
+    ffi.cdef("extern int fooarray[];")
     lib = ffi.verify("int fooarray[50];")
     assert repr(lib.fooarray).startswith("<cdata 'int *'")
 
 def test_global_array_with_dotdotdot_length():
     ffi = FFI()
-    ffi.cdef("int fooarray[...];")
+    ffi.cdef("extern int fooarray[...];")
     lib = ffi.verify("int fooarray[50];")
     assert repr(lib.fooarray).startswith("<cdata 'int[50]'")
 
@@ -1658,7 +1646,7 @@
     py.test.xfail("was detected only because 23 bytes cannot be divided by 4; "
                   "redo more generally")
     ffi = FFI()
-    ffi.cdef("int fooarray[...];")
+    ffi.cdef("extern int fooarray[...];")
     py.test.raises(VerificationError, ffi.verify, "char fooarray[23];")
 
 def test_struct_containing_struct():
@@ -1779,7 +1767,7 @@
 def test_callback_indirection():
     ffi = FFI()
     ffi.cdef("""
-        int (*python_callback)(int how_many, int *values);
+        static int (*python_callback)(int how_many, int *values);
         int (*const c_callback)(int,...);   /* pass this ptr to C routines */
         int some_c_function(int(*cb)(int,...));
     """)
@@ -1913,23 +1901,23 @@
 
 def test_bug_const_char_ptr_array_1():
     ffi = FFI()
-    ffi.cdef("""const char *a[...];""")
+    ffi.cdef("""extern const char *a[...];""")
     lib = ffi.verify("""const char *a[5];""")
     assert repr(ffi.typeof(lib.a)) == "<ctype 'char *[5]'>"
 
 def test_bug_const_char_ptr_array_2():
     ffi = FFI()
-    ffi.cdef("""const int a[];""")
+    ffi.cdef("""extern const int a[];""")
     lib = ffi.verify("""const int a[5];""")
     assert repr(ffi.typeof(lib.a)) == "<ctype 'int *'>"
 
 def _test_various_calls(force_libffi):
     cdef_source = """
-    int xvalue;
-    long long ivalue, rvalue;
-    float fvalue;
-    double dvalue;
-    long double Dvalue;
+    extern int xvalue;
+    extern long long ivalue, rvalue;
+    extern float fvalue;
+    extern double dvalue;
+    extern long double Dvalue;
     signed char tf_bb(signed char x, signed char c);
     unsigned char tf_bB(signed char x, unsigned char c);
     short tf_bh(signed char x, short c);
@@ -2112,7 +2100,7 @@
     old = sys.getdlopenflags()
     try:
         ffi1 = FFI()
-        ffi1.cdef("int foo_verify_dlopen_flags_1;")
+        ffi1.cdef("extern int foo_verify_dlopen_flags_1;")
         sys.setdlopenflags(ffi1.RTLD_GLOBAL | ffi1.RTLD_NOW)
         lib1 = ffi1.verify("int foo_verify_dlopen_flags_1;")
     finally:
@@ -2253,7 +2241,7 @@
 
 def test_macro_var():
     ffi = FFI()
-    ffi.cdef("int myarray[50], my_value;")
+    ffi.cdef("extern int myarray[50], my_value;")
     lib = ffi.verify("""
         int myarray[50];
         int *get_my_value(void) {
diff --git a/extra_tests/cffi_tests/embedding/add1.py b/extra_tests/cffi_tests/embedding/add1.py
--- a/extra_tests/cffi_tests/embedding/add1.py
+++ b/extra_tests/cffi_tests/embedding/add1.py
@@ -12,7 +12,11 @@
     sys.stdout.write("preparing")
     for i in range(3):
         sys.stdout.flush()
-        time.sleep(0.2)
+        # Windows: sometimes time.sleep() doesn't sleep at all.
+        # This appears to occur on recent versions of python only.
+        t_end = time.time() + 0.19
+        while time.time() < t_end:
+            time.sleep(0.2)
         sys.stdout.write(".")
     sys.stdout.write("\n")
 
diff --git a/extra_tests/cffi_tests/embedding/add_recursive.py b/extra_tests/cffi_tests/embedding/add_recursive.py
--- a/extra_tests/cffi_tests/embedding/add_recursive.py
+++ b/extra_tests/cffi_tests/embedding/add_recursive.py
@@ -4,7 +4,7 @@
 ffi = cffi.FFI()
 
 ffi.embedding_api("""
-    int (*my_callback)(int);
+    extern int (*my_callback)(int);
     int add_rec(int, int);
 """)
 
diff --git a/extra_tests/cffi_tests/embedding/test_thread.py b/extra_tests/cffi_tests/embedding/test_thread.py
--- a/extra_tests/cffi_tests/embedding/test_thread.py
+++ b/extra_tests/cffi_tests/embedding/test_thread.py
@@ -22,17 +22,21 @@
         add1_cffi = self.prepare_module('add1')
         add2_cffi = self.prepare_module('add2')
         self.compile('thread2-test', [add1_cffi, add2_cffi], threads=True)
-        output = self.execute('thread2-test')
-        output = self._take_out(output, "preparing")
-        output = self._take_out(output, ".")
-        output = self._take_out(output, ".")
-        # at least the 3rd dot should be after everything from ADD2
-        assert output == ("starting\n"
-                          "prepADD2\n"
-                          "adding 1000 and 200 and 30\n"
-                          ".\n"
-                          "adding 40 and 2\n"
-                          "done\n")
+        for i in range(3):
+            output = self.execute('thread2-test')
+            print('='*79)
+            print(output)
+            print('='*79)
+            output = self._take_out(output, "preparing")
+            output = self._take_out(output, ".")
+            output = self._take_out(output, ".")
+            # at least the 3rd dot should be after everything from ADD2
+            assert output == ("starting\n"
+                              "prepADD2\n"
+                              "adding 1000 and 200 and 30\n"
+                              ".\n"
+                              "adding 40 and 2\n"
+                              "done\n")
 
     def test_alt_issue(self):
         add1_cffi = self.prepare_module('add1')
diff --git a/extra_tests/cffi_tests/support.py b/extra_tests/cffi_tests/support.py
--- a/extra_tests/cffi_tests/support.py
+++ b/extra_tests/cffi_tests/support.py
@@ -1,5 +1,5 @@
 # Generated by pypy/tool/import_cffi.py
-import sys
+import sys, os
 
 if sys.version_info < (3,):
     __all__ = ['u']
@@ -87,3 +87,24 @@
         if not name.startswith('_') and not hasattr(module.ffi, name):
             setattr(ffi, name, NotImplemented)
     return module.lib
+
+
+# For testing, we call gcc with "-Werror".  This is fragile because newer
+# versions of gcc are always better at producing warnings, particularly for
+# auto-generated code.  We need here to adapt and silence them as needed.
+
+if sys.platform == 'win32':
+    extra_compile_args = []      # no obvious -Werror equivalent on MSVC
+else:
+    if (sys.platform == 'darwin' and
+          [int(x) for x in os.uname()[2].split('.')] >= [11, 0, 0]):
+        # assume a standard clang or gcc
+        extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion',
+                              '-Wno-unreachable-code']
+        # special things for clang
+        extra_compile_args.append('-Qunused-arguments')
+    else:
+        # assume a standard gcc
+        extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion',
+                              '-Wno-unused-parameter',
+                              '-Wno-unreachable-code']
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -86,7 +86,7 @@
             arch = platform.machine()
         g['LDSHARED'] += ' -undefined dynamic_lookup'
         g['CC'] += ' -arch %s' % (arch,)
-        g['MACOSX_DEPLOYMENT_TARGET'] = '10.14'
+        g['MACOSX_DEPLOYMENT_TARGET'] = '10.7'
 
     global _config_vars
     _config_vars = g
diff --git a/lib-python/2.7/ensurepip/__init__.py b/lib-python/2.7/ensurepip/__init__.py
--- a/lib-python/2.7/ensurepip/__init__.py
+++ b/lib-python/2.7/ensurepip/__init__.py
@@ -12,9 +12,9 @@
 __all__ = ["version", "bootstrap"]
 
 
-_SETUPTOOLS_VERSION = "28.8.0"
+_SETUPTOOLS_VERSION = "41.2.0"
 
-_PIP_VERSION = "9.0.1"
+_PIP_VERSION = "19.2.3"
 
 _PROJECTS = [
     ("setuptools", _SETUPTOOLS_VERSION),
@@ -28,8 +28,8 @@
         sys.path = additional_paths + sys.path
 
     # Install the bundled software
-    import pip
-    pip.main(args)
+    import pip._internal
+    return pip._internal.main(args)
 
 
 def version():
diff --git a/lib-python/2.7/ensurepip/_bundled/pip-19.2.3-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-19.2.3-py2.py3-none-any.whl
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..8118df8ac1940f8c6cb410fbc18e5fae59872b95
GIT binary patch

[cut]
diff --git a/lib-python/2.7/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl
deleted file mode 100644
index 4b8ecc69db7e37fc6dd7b6dd8f690508f42866a1..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch

[cut]

diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl
deleted file mode 100644
index 502e3cb418c154872ad6e677ef8b63557b38ec35..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch

[cut]

diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-41.2.0-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-41.2.0-py2.py3-none-any.whl
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..82df6f63f4ee97380af0a29d8825ae775333b86d
GIT binary patch

[cut]

diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py
--- a/lib-python/2.7/sysconfig.py
+++ b/lib-python/2.7/sysconfig.py
@@ -382,6 +382,11 @@
     vars['EXE'] = '.exe'
     vars['VERSION'] = _PY_VERSION_SHORT_NO_DOT
     vars['BINDIR'] = os.path.dirname(_safe_realpath(sys.executable))
+    # pypy only: give us control over the ABI tag in a wheel name
+    if '__pypy__' in sys.builtin_module_names:
+        import imp
+        so_ext = imp.get_suffixes()[0][0]
+        vars['SOABI']= '-'.join(so_ext.split('.')[1].split('-')[:2])
 
 #
 # public APIs
diff --git a/lib-python/2.7/test/test_dictviews.py b/lib-python/2.7/test/test_dictviews.py
--- a/lib-python/2.7/test/test_dictviews.py
+++ b/lib-python/2.7/test/test_dictviews.py
@@ -182,7 +182,7 @@
 
     def test_deeply_nested_repr(self):
         d = {}
-        for i in range(sys.getrecursionlimit() + 100):
+        for i in range(sys.getrecursionlimit() + 200):
             d = {42: d.viewvalues()}
         self.assertRaises(RuntimeError, repr, d)
 
diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -2,8 +2,18 @@
 import time
 import thread as _thread
 import weakref
-from _pypy_openssl import ffi
-from _pypy_openssl import lib
+
+try:
+    from _pypy_openssl import ffi
+    from _pypy_openssl import lib
+except ImportError as e:
+    import os
+    msg = "\n\nThe _ssl cffi module either doesn't exist or is incompatible with your machine's shared libraries.\n" + \
+          "If you have a compiler installed, you can try to rebuild it by running:\n" + \
+          "cd %s\n" % os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) + \
+          "%s _ssl_build.py\n" % sys.executable
+    raise ImportError(str(e) + msg)
+
 from _cffi_ssl._stdssl.certificate import (_test_decode_cert,
     _decode_certificate, _certificate_to_der)
 from _cffi_ssl._stdssl.utility import (_str_with_len, _bytes_with_len,
diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py
--- a/lib_pypy/_ctypes/array.py
+++ b/lib_pypy/_ctypes/array.py
@@ -267,6 +267,8 @@
         raise TypeError("Can't multiply a ctypes type by a non-integer")
     if length < 0:
         raise ValueError("Array length must be >= 0")
+    if length * base._sizeofinstances() > sys.maxsize:
+        raise OverflowError("array too large")
     key = (base, length)
     try:
         return ARRAY_CACHE[key]
diff --git a/lib_pypy/_curses_build.py b/lib_pypy/_curses_build.py
--- a/lib_pypy/_curses_build.py
+++ b/lib_pypy/_curses_build.py
@@ -1,18 +1,42 @@
-from cffi import FFI
+from cffi import FFI, VerificationError
 import os
 
-# On some systems, the ncurses library is
-# located at /usr/include/ncurses, so we must check this case.
-# Let's iterate over well known paths
-incdirs =  []
-for _path in ['/usr/include', '/usr/include/ncurses']:
-    if os.path.isfile(os.path.join(_path, 'panel.h')):
-        incdirs.append(_path)
-        break
+version_str = '''
+    static const int NCURSES_VERSION_MAJOR;
+    static const int NCURSES_VERSION_MINOR;
+'''
+
+version = (0, 0)
+def find_library(options):
+    global version
+    for library in options:
+        ffi = FFI()
+        ffi.cdef(version_str)
+        ffi.set_source("_curses_cffi_check", version_str, libraries=[library])
+        try:
+            ffi.compile()
+            import _curses_cffi_check
+            lib = _curses_cffi_check.lib
+            version = (lib.NCURSES_VERSION_MAJOR, lib.NCURSES_VERSION_MINOR)
+        except VerificationError as e:
+            e_last = e
+            continue
+        else:
+            return library
+
+    # If none of the options is available, present the user a meaningful
+    # error message
+    raise e_last
+
+def find_curses_include_dirs():
+    if os.path.exists('/usr/include/ncurses'):
+        return ['/usr/include/ncurses']
+    if os.path.exists('/usr/include/ncursesw'):
+        return ['/usr/include/ncursesw']
+    return []
 
 
 ffi = FFI()
-
 ffi.set_source("_curses_cffi", """
 #ifdef __APPLE__
 /* the following define is necessary for OS X 10.6+; without it, the
@@ -59,8 +83,9 @@
 void _m_getsyx(int *yx) {
     getsyx(yx[0], yx[1]);
 }
-""", include_dirs=incdirs, 
-     libraries=['ncurses', 'panel'])
+""", libraries=[find_library(['ncurses', 'ncursesw']),
+                find_library(['panel', 'panelw'])],
+     include_dirs=find_curses_include_dirs())
 
 
 ffi.cdef("""
@@ -82,6 +107,7 @@
 static const int ERR, OK;
 static const int TRUE, FALSE;
 static const int KEY_MIN, KEY_MAX;
+static const int KEY_CODE_YES;
 
 static const int COLOR_BLACK;
 static const int COLOR_RED;
@@ -141,11 +167,11 @@
 
 int setupterm(char *, int, int *);
 
-WINDOW *stdscr;
-int COLORS;
-int COLOR_PAIRS;
-int COLS;
-int LINES;
+extern WINDOW *stdscr;
+extern int COLORS;
+extern int COLOR_PAIRS;
+extern int COLS;
+extern int LINES;
 
 int baudrate(void);
 int beep(void);
@@ -321,7 +347,7 @@
 #define _m_NetBSD ...
 int _m_ispad(WINDOW *);
 
-chtype acs_map[];
+extern chtype acs_map[];
 
 // For _curses_panel:
 
@@ -346,6 +372,14 @@
 void _m_getsyx(int *yx);
 """)
 
+if version > (5, 7):
+    ffi.cdef("""
+typedef int... wint_t;
+int wget_wch(WINDOW *, wint_t *);
+int mvwget_wch(WINDOW *, int, int, wint_t *);
+int unget_wch(const wchar_t);
+""")
+
 
 if __name__ == "__main__":
     ffi.compile()
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.13.0
+Version: 1.13.2
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -5,8 +5,8 @@
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 from .error import PkgConfigError
 
-__version__ = "1.13.0"
-__version_info__ = (1, 13, 0)
+__version__ = "1.13.2"
+__version_info__ = (1, 13, 2)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -261,12 +261,12 @@
         return (int)_cffi_to_c_wchar3216_t(o);
 }
 
-_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char32_t(int x)
+_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char32_t(unsigned int x)
 {
     if (sizeof(_cffi_wchar_t) == 4)
         return _cffi_from_c_wchar_t((_cffi_wchar_t)x);
     else
-        return _cffi_from_c_wchar3216_t(x);
+        return _cffi_from_c_wchar3216_t((int)x);
 }
 
 
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -224,7 +224,7 @@
 
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.13.0"
+                               "\ncompiled with cffi version: 1.13.2"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
@@ -327,13 +327,15 @@
 #endif
 
     /* call Py_InitializeEx() */
-    {
-        PyGILState_STATE state = PyGILState_UNLOCKED;
-        if (!Py_IsInitialized())
-            _cffi_py_initialize();
-        else
-            state = PyGILState_Ensure();
-
+    if (!Py_IsInitialized()) {
+        _cffi_py_initialize();
+        PyEval_InitThreads();
+        PyEval_SaveThread();  /* release the GIL */
+        /* the returned tstate must be the one that has been stored into the
+           autoTLSkey by _PyGILState_Init() called from Py_Initialize(). */
+    }
+    else {
+        PyGILState_STATE state = PyGILState_Ensure();
         PyEval_InitThreads();
         PyGILState_Release(state);
     }
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -156,6 +156,13 @@
                           "confuse pre-parsing.")
             break
 
+def _warn_for_non_extern_non_static_global_variable(decl):
+    if not decl.storage:
+        import warnings
+        warnings.warn("Global variable '%s' in cdef(): for consistency "
+                      "with C it should have a storage class specifier "
+                      "(usually 'extern')" % (decl.name,))
+
 def _preprocess(csource):
     # Remove comments.  NOTE: this only work because the cdef() section
     # should not contain any string literal!
@@ -506,6 +513,7 @@
                     if (quals & model.Q_CONST) and not tp.is_array_type:
                         self._declare('constant ' + decl.name, tp, quals=quals)
                     else:
+                        _warn_for_non_extern_non_static_global_variable(decl)
                         self._declare('variable ' + decl.name, tp, quals=quals)
 
     def parse_type(self, cdecl):
diff --git a/lib_pypy/pyrepl/historical_reader.py b/lib_pypy/pyrepl/historical_reader.py
--- a/lib_pypy/pyrepl/historical_reader.py
+++ b/lib_pypy/pyrepl/historical_reader.py
@@ -111,7 +111,7 @@
         r.isearch_term = ''
         r.dirty = 1
         r.push_input_trans(r.isearch_trans)
-        
+
 
 class reverse_history_isearch(commands.Command):
     def do(self):
@@ -217,7 +217,7 @@
         self.isearch_trans = input.KeymapTranslator(
             isearch_keymap, invalid_cls=isearch_end,
             character_cls=isearch_add_character)
-        
+
     def select_item(self, i):
         self.transient_history[self.historyi] = self.get_unicode()
         buf = self.transient_history.get(i)
@@ -302,7 +302,7 @@
     reader.ps1 = "h**> "
     reader.ps2 = "h/*> "
     reader.ps3 = "h|*> "
-    reader.ps4 = "h\*> "
+    reader.ps4 = r"h\*> "
     while reader.readline():
         pass
 
diff --git a/lib_pypy/pyrepl/keymap.py b/lib_pypy/pyrepl/keymap.py
--- a/lib_pypy/pyrepl/keymap.py
+++ b/lib_pypy/pyrepl/keymap.py
@@ -58,7 +58,7 @@
     "'":"'",
     '"':'"',
     'a':'\a',
-    'b':'\h',
+    'b':'\b',
     'e':'\033',
     'f':'\f',
     'n':'\n',
diff --git a/lib_pypy/pyrepl/reader.py b/lib_pypy/pyrepl/reader.py
--- a/lib_pypy/pyrepl/reader.py
+++ b/lib_pypy/pyrepl/reader.py
@@ -232,7 +232,7 @@
         self.ps1 = "->> "
         self.ps2 = "/>> "
         self.ps3 = "|.. "
-        self.ps4 = "\__ "
+        self.ps4 = r"\__ "
         self.kill_ring = []
         self.arg = None
         self.finished = 0
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -37,7 +37,7 @@
     "binascii", "_multiprocessing", '_warnings', "_collections",
     "_multibytecodec", "micronumpy", "_continuation", "_cffi_backend",
     "_csv", "_cppyy", "_pypyjson", "_jitlog",
-    #" _ssl", "_hashlib", "crypt"
+    # "_hashlib", "crypt"
 ])
 
 import rpython.rlib.rvmprof.cintf
diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -114,8 +114,8 @@
         return PyPyModule(path, parent)
 
 def is_applevel(item):
-    from pypy.tool.pytest.apptest import AppTestFunction
-    return isinstance(item, AppTestFunction)
+    from pypy.tool.pytest.apptest import AppTestMethod
+    return isinstance(item, AppTestMethod)
 
 def pytest_collection_modifyitems(config, items):
     if config.getoption('runappdirect') or config.getoption('direct_apptest'):
@@ -145,8 +145,6 @@
     def funcnamefilter(self, name):
         if name.startswith('test_'):
             return self.accept_regular_test()
-        if name.startswith('app_test_'):
-            return True
         return False
 
     def classnamefilter(self, name):
@@ -161,13 +159,6 @@
             if name.startswith('AppTest'):
                 from pypy.tool.pytest.apptest import AppClassCollector
                 return AppClassCollector(name, parent=self)
-
-        elif hasattr(obj, 'func_code') and self.funcnamefilter(name):
-            if name.startswith('app_test_'):
-                assert not obj.func_code.co_flags & 32, \
-                    "generator app level functions? you must be joking"
-                from pypy.tool.pytest.apptest import AppTestFunction
-                return AppTestFunction(name, parent=self)
         return super(PyPyModule, self).makeitem(name, obj)
 
 def skip_on_missing_buildoption(**ropts):
@@ -186,27 +177,18 @@
     py.test.skip("need translated pypy with: %s, got %s"
                  %(ropts,options))
 
-class LazyObjSpaceGetter(object):
-    def __get__(self, obj, cls=None):
-        from pypy.tool.pytest.objspace import gettestobjspace
-        space = gettestobjspace()
-        if cls:
-            cls.space = space
-        return space
-
-
 @pytest.hookimpl(tryfirst=True)
 def pytest_runtest_setup(item):
     if isinstance(item, py.test.collect.Function):
         appclass = item.getparent(py.test.Class)
         if appclass is not None:
+            from pypy.tool.pytest.objspace import gettestobjspace
             # Make cls.space and cls.runappdirect available in tests.
-            spaceconfig = getattr(appclass.obj, 'spaceconfig', None)
-            if spaceconfig is not None:
-                from pypy.tool.pytest.objspace import gettestobjspace
-                appclass.obj.space = gettestobjspace(**spaceconfig)
-            else:
-                appclass.obj.space = LazyObjSpaceGetter()
+            spaceconfig = getattr(appclass.obj, 'spaceconfig', {})
+            config = item.config
+            if not (config.getoption('runappdirect') or config.getoption('direct_apptest')):
+                spaceconfig.setdefault('objspace.std.reinterpretasserts', True)
+            appclass.obj.space = gettestobjspace(**spaceconfig)
             appclass.obj.runappdirect = option.runappdirect
 
 def pytest_ignore_collect(path, config):
diff --git a/pypy/doc/commandline_ref.rst b/pypy/doc/commandline_ref.rst
--- a/pypy/doc/commandline_ref.rst
+++ b/pypy/doc/commandline_ref.rst
@@ -9,3 +9,4 @@
 
    man/pypy.1.rst
    man/pypy3.1.rst
+   jit_help.rst
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -71,9 +71,9 @@
 #    module/cpyext/include/patchlevel.h
 #
 # The short X.Y version.
-version = '7.2'
+version = '7.3'
 # The full version, including alpha/beta/rc tags.
-release = '7.2.0'
+release = '7.3.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -42,7 +42,9 @@
 -------------------------------
 
 .. toctree::
-   whatsnew-pypy3-head.rst
+    whatsnew-pypy3-head.rst
+    whatsnew-pypy3-7.2.0.rst
+    whatsnew-pypy3-7.1.0.rst
 
 CPython 3.5 compatible versions
 -------------------------------
@@ -50,6 +52,8 @@
 .. toctree::
 
    whatsnew-pypy3-7.0.0.rst
+   whatsnew-pypy3-6.0.0.rst
+   whatsnew-pypy3-5.10.0.rst
    whatsnew-pypy3-5.9.0.rst
    whatsnew-pypy3-5.8.0.rst
    whatsnew-pypy3-5.7.0.rst
@@ -62,10 +66,4 @@
    whatsnew-pypy3-5.5.0.rst
    whatsnew-pypy3-5.1.1-alpha1.rst
 
-CPython 3.2 compatible versions
--------------------------------
 
-.. toctree::
-
-   whatsnew-pypy3-2.4.0.rst
-   whatsnew-pypy3-2.3.1.rst
diff --git a/pypy/doc/jit_help.rst b/pypy/doc/jit_help.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/jit_help.rst
@@ -0,0 +1,78 @@
+========
+JIT help
+========
+
+.. note this is from ``pypy --jit help``
+
+Advanced JIT options
+====================
+
+``<pypy> --jit`` [*options*] where *options* is a comma-separated list of
+``OPTION=VALUE``:
+
+ decay=N
+    amount to regularly decay counters by (0=none, 1000=max) (default 40)
+
+ disable_unrolling=N
+    after how many operations we should not unroll (default 200)
+
+ enable_opts=N
+    INTERNAL USE ONLY (MAY NOT WORK OR LEAD TO CRASHES): optimizations to
+    enable, or all =
+    intbounds:rewrite:virtualize:string:pure:earlyforce:heap:unroll (default
+    all)
+
+ function_threshold=N
+    number of times a function must run for it to become traced from start
+    (default 1619)
+
+ inlining=N
+    inline python functions or not (1/0) (default 1)
+
+ loop_longevity=N
+    a parameter controlling how long loops will be kept before being freed,
+    an estimate (default 1000)
+
+ max_retrace_guards=N
+    number of extra guards a retrace can cause (default 15)
+
+ max_unroll_loops=N
+    number of extra unrollings a loop can cause (default 0)
+
+ max_unroll_recursion=N
+    how many levels deep to unroll a recursive function (default 7)
+
+ retrace_limit=N
+    how many times we can try retracing before giving up (default 0)
+
+ threshold=N
+    number of times a loop has to run for it to become hot (default 1039)
+
+ trace_eagerness=N
+    number of times a guard has to fail before we start compiling a bridge
+    (default 200)
+
+ trace_limit=N
+    number of recorded operations before we abort tracing with ABORT_TOO_LONG
+    (default 6000)
+
+ vec=N
+    turn on the vectorization optimization (vecopt). Supports x86 (SSE 4.1),
+    powerpc (SVX), s390x SIMD (default 0)
+
+ vec_all=N
+    try to vectorize trace loops that occur outside of the numpypy library
+    (default 0)
+
+ vec_cost=N
+    threshold for which traces to bail. Unpacking increases the counter,
+    vector operation decrease the cost (default 0)
+
+ off
+    turn off the JIT
+ help
+    print this page
+
+The :ref:`pypyjit<jit-hooks>` module can be used to control the JIT from inside
+pypy
+
diff --git a/pypy/doc/release-v7.2.0.rst b/pypy/doc/release-v7.2.0.rst
--- a/pypy/doc/release-v7.2.0.rst
+++ b/pypy/doc/release-v7.2.0.rst
@@ -216,6 +216,7 @@
 * Add more constants to `sysconfig``. Set ``MACOSX_DEPLOYMENT_TARGET`` for
   darwin (`issue 2994`_)
 * fix ``CBuffer.buffer_attach``
+* Add ``_PyDict_GetItemWithError`` (``PyDict_GetItemWithError`` on Python3)
 
 Python 3.6 only
 ---------------
@@ -307,6 +308,7 @@
 .. _33786 : https://bugs.python.org/issue33786
 .. _32270 : https://bugs.python.org/issue32270
 .. _28691 : https://bugs.python.org/issue28691
+.. _33729 : https://bugs.python.org/issue33729
 
 .. _opencv2: https://github.com/skvark/opencv-python/
 .. _`issue 2617`: https://bitbucket.com/pypy/pypy/issues/2617
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -3,11 +3,31 @@
 ==========================
 
 .. this is a revision shortly after release-pypy-7.2.0
-.. startrev: 78cd4acbcbec 
+.. startrev: a511d86377d6 
 
+.. branch: fix-descrmismatch-crash
 
-.. branch: json-decoder-maps
+Fix segfault when calling descr-methods with no arguments
 
-Much faster and more memory-efficient JSON decoding. The resulting
-dictionaries that come out of the JSON decoder have faster lookups too.
+.. branch: https-readme
 
+Convert http -> https in README.rst
+
+.. branch: license-update
+
+Update list directories in LICENSE
+
+.. branch: allow-forcing-no-embed
+
+When packaging, allow suppressing embedded dependencies via
+PYPY_NO_EMBED_DEPENDENCIES
+
+.. branch: int-test-is-zero
+
+.. branch: cppyy-dev
+
+Upgraded the built-in ``_cppyy`` module to ``cppyy-backend 1.10.6``, which
+provides, among others, better template resolution, stricter ``enum`` handling,
+anonymous struct/unions, cmake fragments for distribution, optimizations for
+PODs, and faster wrapper calls.
+
diff --git a/pypy/doc/whatsnew-pypy2-5.7.0.rst b/pypy/doc/whatsnew-pypy2-5.7.0.rst
--- a/pypy/doc/whatsnew-pypy2-5.7.0.rst
+++ b/pypy/doc/whatsnew-pypy2-5.7.0.rst
@@ -14,6 +14,7 @@
 changed in exactly the same way because of this fix.
 
 
+
 .. branch: rpython-error-to-systemerror
 
 Any uncaught RPython exception (from a PyPy bug) is turned into an
diff --git a/pypy/doc/whatsnew-pypy2-7.2.0.rst b/pypy/doc/whatsnew-pypy2-7.2.0.rst
--- a/pypy/doc/whatsnew-pypy2-7.2.0.rst
+++ b/pypy/doc/whatsnew-pypy2-7.2.0.rst
@@ -74,3 +74,9 @@
 .. branch: openssl-for-macos
 
 Update _ssl on macos to statically link to openssl-1.1.1c
+
+.. branch: json-decoder-maps
+
+Much faster and more memory-efficient JSON decoding. The resulting
+dictionaries that come out of the JSON decoder have faster lookups too.
+
diff --git a/pypy/doc/whatsnew-pypy3-2.3.1.rst b/pypy/doc/whatsnew-pypy3-2.3.1.rst
deleted file mode 100644
--- a/pypy/doc/whatsnew-pypy3-2.3.1.rst
+++ /dev/null
@@ -1,6 +0,0 @@
-=========================
-What's new in PyPy3 2.3.1
-=========================
-
-.. this is a revision shortly after pypy3-release-2.3.x
-.. startrev: 0137d8e6657d
diff --git a/pypy/doc/whatsnew-pypy3-2.4.0.rst b/pypy/doc/whatsnew-pypy3-2.4.0.rst
deleted file mode 100644
--- a/pypy/doc/whatsnew-pypy3-2.4.0.rst
+++ /dev/null
@@ -1,6 +0,0 @@
-=========================
-What's new in PyPy3 2.4.0
-=========================
-
-.. this is a revision shortly after pypy3-release-2.4.x
-.. startrev: 12b940544622
diff --git a/pypy/doc/whatsnew-pypy3-5.10.0.rst b/pypy/doc/whatsnew-pypy3-5.10.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy3-5.10.0.rst
@@ -0,0 +1,7 @@
+========================
+What's new in PyPy3 7.0+
+========================
+
+.. this is the revision after release-pypy3.5-v7.0
+.. startrev: 9d2fa7c63b7c
+
diff --git a/pypy/doc/whatsnew-pypy3-6.0.0.rst b/pypy/doc/whatsnew-pypy3-6.0.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy3-6.0.0.rst
@@ -0,0 +1,7 @@
+========================
+What's new in PyPy3 7.0+
+========================
+
+.. this is the revision after release-pypy3.5-v7.0
+.. startrev: 9d2fa7c63b7c
+
diff --git a/pypy/doc/whatsnew-pypy3-7.1.0.rst b/pypy/doc/whatsnew-pypy3-7.1.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy3-7.1.0.rst
@@ -0,0 +1,11 @@
+========================
+What's new in PyPy3 7.0+
+========================
+
+.. this is the revision after release-pypy3.6-v7.0
+.. startrev: 33fe3b2cf186
+
+.. branch: py3.5
+
+Merge in py.35 and use this branch as the primary pypy3 one
+
diff --git a/pypy/doc/whatsnew-pypy3-7.2.0.rst b/pypy/doc/whatsnew-pypy3-7.2.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy3-7.2.0.rst
@@ -0,0 +1,58 @@
+=========================
+What's new in PyPy3 7.2.0
+=========================
+
+.. this is the revision after release-pypy3.6-v7.1.1
+.. startrev: db5a1e7fbbd0
+
+.. branch: fix-literal-prev_digit-underscore
+
+Fix parsing for converting strings with underscore into ints
+
+.. branch: winmultiprocessing
+
+Improve multiprocessing support on win32
+
+.. branch: setitem2d
+
+Allow 2d indexing in ``memoryview.__setitem__`` (issue bb-3028)
+
+.. branch: py3.6-socket-fix
+.. branch: fix-importerror
+.. branch: dj_s390
+.. branch: bpo-35409
+.. branch: remove_array_with_char_test
+.. branch: fix_test_unicode_outofrange
+.. branch: Ram-Rachum/faulthandleris_enabled-should-return-fal-1563636614875
+.. branch: Anthony-Sottile/fix-leak-of-file-descriptor-with-_iofile-1559687440863
+
+.. branch: py3tests
+
+Add handling of application-level test files and -D flag to test runner
+
+.. branch: vendor/stdlib-3.6
+.. branch: stdlib-3.6.9
+
+Update standard library to version 3.6.9
+
+.. branch: __debug__-optimize
+
+Fix handling of __debug__, sys.flags.optimize, and '-O' command-line flag to 
+match CPython 3.6.
+
+.. branch: more-cpyext
+
+Add ``PyErr_SetFromWindowsErr`` and ``pytime.h``, ``pytime.c``. Fix order of
+fields in ``Py_buffer``.
+
+.. branch: Ryan-Hileman/add-support-for-zipfile-stdlib-1562420744699
+
+Add support for the entire stdlib being inside a zipfile
+
+
+.. branch: json-decoder-maps-py3.6
+
+Much faster and more memory-efficient JSON decoding. The resulting
+dictionaries that come out of the JSON decoder have faster lookups too.
+
+
diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -1,7 +1,7 @@
-========================
-What's new in PyPy3 7.0+
-========================
-
-.. this is the revision after release-pypy3.5-v7.0
-.. startrev: 9d2fa7c63b7c
-
+========================
+What's new in PyPy3 7.2+
+========================
+
+.. this is the revision after release-pypy3.6-v7.2
+.. startrev: 6d2f8470165b
+
diff --git a/pypy/interpreter/astcompiler/test/apptest_misc.py b/pypy/interpreter/astcompiler/test/apptest_misc.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/astcompiler/test/apptest_misc.py
@@ -0,0 +1,18 @@
+def test_warning_to_error_translation():
+    import warnings
+    statement = """\
+def wrong1():
+    a = 1
+    b = 2
+    global a
+    global b
+"""
+    with warnings.catch_warnings():
+        warnings.filterwarnings("error", module="<test string>")
+        try:
+           compile(statement, '<test string>', 'exec')
+        except SyntaxError as err:
+           assert err.lineno is not None
+           assert err.filename is not None
+           assert err.offset is not None
+           assert err.message is not None
diff --git a/pypy/interpreter/astcompiler/test/test_misc.py b/pypy/interpreter/astcompiler/test/test_misc.py
--- a/pypy/interpreter/astcompiler/test/test_misc.py
+++ b/pypy/interpreter/astcompiler/test/test_misc.py
@@ -12,23 +12,3 @@
     assert mangle("__foo", "__Bar") == "_Bar__foo"
     assert mangle("__foo", "___") == "__foo"
     assert mangle("___foo", "__Bar") == "_Bar___foo"
-
-def app_test_warning_to_error_translation():
-    import warnings
-    
-    with warnings.catch_warnings():
-        warnings.filterwarnings("error", module="<test string>")
-        statement = """\
-def wrong1():
-    a = 1
-    b = 2
-    global a
-    global b
-"""
-        try:
-           compile(statement, '<test string>', 'exec')
-        except SyntaxError as err:
-           assert err.lineno is not None
-           assert err.filename is not None
-           assert err.offset is not None
-           assert err.message is not None
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py

From pypy.commits at gmail.com  Wed Nov 27 02:01:44 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 26 Nov 2019 23:01:44 -0800 (PST)
Subject: [pypy-commit] pypy default: restart whatsnew, add release note
Message-ID: <5dde1f58.1c69fb81.46a32.9221@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r98157:78cf24b82246
Date: 2019-11-26 08:33 -0800
http://bitbucket.org/pypy/pypy/changeset/78cf24b82246/

Log:	restart whatsnew, add release note

diff --git a/pypy/doc/release-v7.3.0.rst b/pypy/doc/release-v7.3.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-v7.3.0.rst
@@ -0,0 +1,213 @@
+====================================
+PyPy v7.3.0: release of 2.7, and 3.6
+====================================
+
+The PyPy team is proud to release the version 7.3.0 of PyPy, which includes
+two different interpreters:
+
+  - PyPy2.7, which is an interpreter supporting the syntax and the features of
+    Python 2.7 including the stdlib for CPython 2.7.13
+
+  - PyPy3.6: which is an interpreter supporting the syntax and the features of
+    Python 3.6, including the stdlib for CPython 3.6.9.
+    
+The interpreters are based on much the same codebase, thus the double
+release.
+
+We have worked with the python packaging group to support tooling around
+building third party packages for python, so this release changes the ABI tag
+for PyPy.
+
+The `CFFI`_ backend has been updated to version 1.13.1. We recommend using CFFI
+rather than c-extensions to interact with C.
+
+The built-in ``_cppyy`` module was upgraded to 1.10.6, which
+provides, among others, better template resolution, stricter ``enum`` handling,
+anonymous struct/unions, cmake fragments for distribution, optimizations for
+PODs, and faster wrapper calls. We reccomend using cppyy_ for performant
+wrapping of C++ code for Python.
+
+The vendored pyrepl package for interaction inside the REPL was updated.
+
+Support for codepage encoding and decoding was added for Windows.
+
+As always, this release fixed several issues and bugs raised by the growing
+community of PyPy users.  We strongly recommend updating. Many of the fixes are
+the direct result of end-user bug reports, so please continue reporting issues
+as they crop up.
+
+You can download the v7.3 releases here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project. If PyPy is not quite good enough for your needs, we are available for
+direct consulting work.
+
+We would also like to thank our contributors and encourage new people to join
+the project. PyPy has many layers and we need help with all of them: `PyPy`_
+and `RPython`_ documentation improvements, tweaking popular modules to run
+on pypy, or general `help`_ with making RPython's JIT even better. Since the
+previous release, we have accepted contributions from 27 new contributors,
+thanks for pitching in.
+
+.. _`PyPy`: index.html
+.. _`RPython`: https://rpython.readthedocs.org
+.. _`help`: project-ideas.html
+.. _`CFFI`: http://cffi.readthedocs.io
+.. _`cppyy`: https://cppyy.readthedocs.io
+.. _`available as wheels`: https://github.com/antocuni/pypy-wheels
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7, 3.6. It's fast (`PyPy and CPython 2.7.x`_ performance
+comparison) due to its integrated tracing JIT compiler.
+
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
+
+This PyPy release supports:
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
+
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+  * 64-bit **ARM** machines running Linux.
+
+Unfortunately at the moment of writing our ARM buildbots are out of service,
+so for now we are **not** releasing any binary for the ARM architecture (32
+bit), although PyPy does support ARM 32 bit processors. 
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html
+
+
+Changelog
+=========
+
+Changes shared across versions
+------------------------------
+
+* Fix segfault when calling descr-methods with no arguments
+* Change the SOABI and subsequently change the reported ABI tag.
+* Update the builtin ``_cppyy`` backend to 1.10.6
+* Performance improvements in string/unicode handling
+* Fix compilation error when building `revdb`_ (`issue 3084`_, actually
+  released in PyPy 7.2 but not mentioned in the release notes)
+* Add JIT options to documentation and an option for JIT help via ``pypy --jit
+  help``
+* Fix regex escapes in pyrepl (`issue 3097`_)
+* Allow reloading of subclasses of module (`issue 3099`_)
+* Work around a gcc bug, which was reported to them and fixed (`issue 3086`_)
+* Fix (again) faulty logic when decoding invalid UTF-8 (`issue 2389`_)
+* Fix up LICENSE file
+* Turn all ``http`` links in the documentation to ``https``
+* Update the bndled pip and setuptools (used in ``pypy -mensurepip`` to version
+  that support `manylinux2010`_ wheels
+* Link the ``DEFAULT_SOABI`` to the ``PYPY_VERSION``
+* Workaround for programs calling ``sys.setrecursionlimit(huge_value)`` (`issue
+  3094`_)
+* Set minimal ``MACOSX_DEPLOYMENT_TARGET`` to 10.7 on macOS; cpython uses 10.5
+* Fix a JIT bug causing rare register corruption on aarch64
+* Add discovery of ``ncursesw`` when building ``_minimal_curses`` and improve
+  handling of old curses versions (`issue 2970`_)
+* Improve the error message for ``class X(random_object)`` (`issue 3109`_)
+* Deal with json dicts containing repeated keys in the new map based parser
+  (`issue 3108`_)
+* Port parts of the `portable pypy`_ repackaging scripts to add an option for
+  ``RPATH`` manipulation on linux
+* Check for overflow in ctypes array creation
+* Better support and report MSVC versions used to compile on windows
+* Allow any kind of buffer in socket.setsockopt(), like CPython (`issue 3114`_)
+
+C-API (cpyext) and c-extensions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* Add ``_PySet_Next``, ``_PySet_NextEntry``
+* Correctly swallow exceptions happening inside ``PyDict_GetItem()`` (`issue
+  3098`_)
+* Respect tp_dict on PyType_Ready (`issue XXX`_)
+* Allow calling ``PyType_Ready`` on a subclass with a partially built
+  ``tp_base`` (issue 3117`_)
+* Rename ``tuple_new`` to ``_PyPy_tuple_new`` to follow the naming convention of
+  exported symbols in ``libpypy-c.so``
+* Actually restore the traceback in ``PyErr_Restore`` (`issue 3120`_)
+
+Python 3.6 only
+---------------
+
+* Don't grow the ``lzma.decompress()`` buffer past ``max_length`` (`issue 3088`_)
+* Backport fix from CPython for failure of ``lzma`` to decompress a file
+  (`issue 3090`_)
+* Fix ``asyncgen_hooks`` and refactor ``coroutine execution``
+* Fix range checking in GB18030 decoder (CPython issue `29990`_)
+* Fix handling escape characters in HZ codec (CPython issue `30003`_)
+* Reject null characters in a few more functions (CPython issue `13617`_)
+* Fix build on macOS without ``clock_gettime`` (before 10.12 and xcode 8,
+  released 2016)
+* Backport 3.7.5 changes to ``timedelta.__eq__`` and ``time.__eq__`` (CPython
+  issue `37579`_)
+* Backport more fixes to comparisons in ``datetime.py`` (CPython issue `37985`_)
+* Use the python tag in ``pyc`` file names, not the abi tag
+* Handle string formatting with a single ``[`` in the format string (`issue
+  3100`_)
+* Backport some of the patches in `macports pypy`_
+* Add missing ``HAVE_FACCESSAT`` to ``posix._have_functions``
+* Update pyrepl from upstream package (`issue 2971`_)
+* Fix ``PyFrame._guess_function_name_parens()``
+* Fix range of allowed years in ``time.mktime`` to match CPython `13312`_
+* Generators need to store the old current ``exc_info`` in a place that is
+  visible, because in one corner case a call to ``sys.exc_info()`` might need
+  it.  (`issue 3096`_)
+* Remove incorrect clobbering of the ``locals`` after running ``exec()``
+* Adds encoding, decoding codepages on win32
+
+Python 3.6 C-API
+~~~~~~~~~~~~~~~~
+
+* Add ``PyObject_GenericGetDict``, ``PyObject_GenericSetDict``, ``_Py_strhex``,
+  ``_Py_strhex_bytes``, ``PyUnicodeNew``, ``_PyFinalizing``,
+  ``PySlice_Unpack``, ``PySlice_AdjustIndices``
+* Implement ``pystrhex.h`` (`issue 2687`_)
+* Make ``PyUnicodeObject`` slightly more compact
+* Fix memory leak when releasing a ``PyUnicodeObject``
+
+.. _`revdb`: fix broken link
+.. _`portable pypy`: fix broken link
+.. _`manylinux2010`: fix broken link
+.. _`macports pypy`: https://github.com/macports/macports-ports/blob/master/lang/pypy/files/darwin.py.diff
+
+.. _`issue 2389`: https://bitbucket.com/pypy/pypy/issues/2389
+.. _`issue 2687`: https://bitbucket.com/pypy/pypy/issues/2687
+.. _`issue 2970`: https://bitbucket.com/pypy/pypy/issues/2970
+.. _`issue 2971`: https://bitbucket.com/pypy/pypy/issues/2971
+.. _`issue 3084`: https://bitbucket.com/pypy/pypy/issues/3084
+.. _`issue 3086`: https://bitbucket.com/pypy/pypy/issues/3086
+.. _`issue 3088`: https://bitbucket.com/pypy/pypy/issues/3088
+.. _`issue 3090`: https://bitbucket.com/pypy/pypy/issues/3090
+.. _`issue 3094`: https://bitbucket.com/pypy/pypy/issues/3094
+.. _`issue 3096`: https://bitbucket.com/pypy/pypy/issues/3096
+.. _`issue 3097`: https://bitbucket.com/pypy/pypy/issues/3097
+.. _`issue 3098`: https://bitbucket.com/pypy/pypy/issues/3098
+.. _`issue 3099`: https://bitbucket.com/pypy/pypy/issues/3099
+.. _`issue 3100`: https://bitbucket.com/pypy/pypy/issues/3100
+.. _`issue 3108`: https://bitbucket.com/pypy/pypy/issues/3108
+.. _`issue 3109`: https://bitbucket.com/pypy/pypy/issues/3109
+.. _`issue 3114`: https://bitbucket.com/pypy/pypy/issues/3114
+.. _`issue 3117`: https://bitbucket.com/pypy/pypy/issues/3117
+.. _`issue 3120`: https://bitbucket.com/pypy/pypy/issues/3120
+
+.. _13312: https://bugs.python.org/issue13312
+.. _13617: https://bugs.python.org/issue13617
+.. _29990: https://bugs.python.org/issue29990
+.. _30003: https://bugs.python.org/issue30003
+.. _37579: https://bugs.python.org/issue37579
+.. _37985: https://bugs.python.org/issue37985
+.. _37985: https://bugs.python.org/issue37985
+
+
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-head.rst
@@ -0,0 +1,8 @@
+============================
+What's new in PyPy2.7 7.3.0+
+============================
+
+.. this is a revision shortly after release-pypy-7.3.0
+.. startrev: dbbbae99135f 
+
+
diff --git a/pypy/doc/whatsnew-pypy2-7.3.0.rst b/pypy/doc/whatsnew-pypy2-7.3.0.rst
--- a/pypy/doc/whatsnew-pypy2-7.3.0.rst
+++ b/pypy/doc/whatsnew-pypy2-7.3.0.rst
@@ -1,6 +1,6 @@
-==========================
-What's new in PyPy2.7 7.3+
-==========================
+===========================
+What's new in PyPy2.7 7.3.0
+===========================
 
 .. this is a revision shortly after release-pypy-7.2.0
 .. startrev: a511d86377d6 
diff --git a/pypy/doc/whatsnew-pypy3-7.3.0.rst b/pypy/doc/whatsnew-pypy3-7.3.0.rst
--- a/pypy/doc/whatsnew-pypy3-7.3.0.rst
+++ b/pypy/doc/whatsnew-pypy3-7.3.0.rst
@@ -1,6 +1,6 @@
-========================
-What's new in PyPy3 7.2+
-========================
+=========================
+What's new in PyPy3 7.3.0
+=========================
 
 .. this is the revision after release-pypy3.6-v7.2
 .. startrev: 6d2f8470165b
diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -0,0 +1,7 @@
+==========================
+What's new in PyPy3 7.3.0+
+==========================
+
+.. this is the revision after release-pypy3.6-v7.3.0
+.. startrev: 78b4d0a7cf2e
+

From pypy.commits at gmail.com  Wed Nov 27 02:01:46 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 26 Nov 2019 23:01:46 -0800 (PST)
Subject: [pypy-commit] pypy default: update contributors
Message-ID: <5dde1f5a.1c69fb81.6ca3f.86d0@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r98158:b7f6c003d6d5
Date: 2019-11-26 08:41 -0800
http://bitbucket.org/pypy/pypy/changeset/b7f6c003d6d5/

Log:	update contributors

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -99,16 +99,16 @@
   Spenser Bauman
   Michal Bendowski
   Jan de Mooij
+  Stefano Rivera
   Tyler Wade
+  Stefan Beyer
   Vincent Legoll
   Michael Foord
   Stephan Diehl
-  Stefano Rivera
   Jean-Paul Calderone
   Stefan Schwarzer
   Tomek Meka
   Valentino Volonghi
-  Stefan Beyer
   Patrick Maupin
   Devin Jeanpierre
   Bob Ippolito
@@ -137,9 +137,10 @@
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
+  Stian Andreassen
+  Julian Berman
   William Leslie
   Pawe&#322; Piotr Przeradowski
-  Stian Andreassen
   marky1991
   Ilya Osadchiy
   Tobias Oberstein
@@ -150,7 +151,7 @@
   tav
   Georg Brandl
   Joannah Nanjekye
-  Julian Berman
+  Yannick Jadoul
   Bert Freudenberg
   Wanja Saatkamp
   Mike Blume
@@ -275,6 +276,7 @@
   Lutz Paelike
   Ian Foote
   Philipp Rustemeuer
+  Bernd Schoeller
   Logan Chien
   Catalin Gabriel Manciu
   Jacob Oscarson
@@ -302,7 +304,6 @@
   Laurens Van Houtven
   Bobby Impollonia
   Roberto De Ioris
-  Yannick Jadoul
   Jeong YunWon
   Christopher Armstrong
   Aaron Tubbs
@@ -357,6 +358,7 @@
   Daniil Yarancev
   Min RK
   OlivierBlanvillain
+  bernd.schoeller at inf.ethz.ch
   dakarpov at gmail.com
   Jonas Pfannschmidt
   Zearin
@@ -398,6 +400,7 @@
   Jesdi
   Konrad Delong
   Dinu Gherman
+  Sam Edwards
   pizi
   Tom&#225;&#353; Pru&#382;ina
   James Robert
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -65,16 +65,16 @@
   Spenser Bauman
   Michal Bendowski
   Jan de Mooij
+  Stefano Rivera
   Tyler Wade
+  Stefan Beyer
   Vincent Legoll
   Michael Foord
   Stephan Diehl
-  Stefano Rivera
   Jean-Paul Calderone
   Stefan Schwarzer
   Tomek Meka
   Valentino Volonghi
-  Stefan Beyer
   Patrick Maupin
   Devin Jeanpierre
   Bob Ippolito
@@ -103,9 +103,10 @@
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
+  Stian Andreassen
+  Julian Berman
   William Leslie
   Pawe&#322; Piotr Przeradowski
-  Stian Andreassen
   marky1991
   Ilya Osadchiy
   Tobias Oberstein
@@ -116,7 +117,7 @@
   tav
   Georg Brandl
   Joannah Nanjekye
-  Julian Berman
+  Yannick Jadoul
   Bert Freudenberg
   Wanja Saatkamp
   Mike Blume
@@ -241,6 +242,7 @@
   Lutz Paelike
   Ian Foote
   Philipp Rustemeuer
+  Bernd Schoeller
   Logan Chien
   Catalin Gabriel Manciu
   Jacob Oscarson
@@ -253,7 +255,6 @@
   Lene Wagner
   Tomo Cocoa
   Miro Hron&#269;ok
-  Anthony Sottile
   David Lievens
   Neil Blakey-Milner
   Henrik Vendelbo
@@ -269,7 +270,6 @@
   Laurens Van Houtven
   Bobby Impollonia
   Roberto De Ioris
-  Yannick Jadoul
   Jeong YunWon
   Christopher Armstrong
   Aaron Tubbs
@@ -324,6 +324,7 @@
   Daniil Yarancev
   Min RK
   OlivierBlanvillain
+  bernd.schoeller at inf.ethz.ch
   dakarpov at gmail.com
   Jonas Pfannschmidt
   Zearin
@@ -365,6 +366,7 @@
   Jesdi
   Konrad Delong
   Dinu Gherman
+  Sam Edwards
   pizi
   Tom&#225;&#353; Pru&#382;ina
   James Robert
diff --git a/pypy/doc/release-v7.3.0.rst b/pypy/doc/release-v7.3.0.rst
--- a/pypy/doc/release-v7.3.0.rst
+++ b/pypy/doc/release-v7.3.0.rst
@@ -48,7 +48,7 @@
 the project. PyPy has many layers and we need help with all of them: `PyPy`_
 and `RPython`_ documentation improvements, tweaking popular modules to run
 on pypy, or general `help`_ with making RPython's JIT even better. Since the
-previous release, we have accepted contributions from 27 new contributors,
+previous release, we have accepted contributions from 3 new contributors,
 thanks for pitching in.
 
 .. _`PyPy`: index.html

From pypy.commits at gmail.com  Wed Nov 27 02:01:48 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 26 Nov 2019 23:01:48 -0800 (PST)
Subject: [pypy-commit] pypy py3.6: merge default into py3.6
Message-ID: <5dde1f5c.1c69fb81.67339.8f08@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r98159:e5d59224adb8
Date: 2019-11-26 08:47 -0800
http://bitbucket.org/pypy/pypy/changeset/e5d59224adb8/

Log:	merge default into py3.6

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -99,16 +99,16 @@
   Spenser Bauman
   Michal Bendowski
   Jan de Mooij
+  Stefano Rivera
   Tyler Wade
+  Stefan Beyer
   Vincent Legoll
   Michael Foord
   Stephan Diehl
-  Stefano Rivera
   Jean-Paul Calderone
   Stefan Schwarzer
   Tomek Meka
   Valentino Volonghi
-  Stefan Beyer
   Patrick Maupin
   Devin Jeanpierre
   Bob Ippolito
@@ -137,9 +137,10 @@
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
+  Stian Andreassen
+  Julian Berman
   William Leslie
   Pawe&#322; Piotr Przeradowski
-  Stian Andreassen
   marky1991
   Ilya Osadchiy
   Tobias Oberstein
@@ -150,7 +151,7 @@
   tav
   Georg Brandl
   Joannah Nanjekye
-  Julian Berman
+  Yannick Jadoul
   Bert Freudenberg
   Wanja Saatkamp
   Mike Blume
@@ -275,6 +276,7 @@
   Lutz Paelike
   Ian Foote
   Philipp Rustemeuer
+  Bernd Schoeller
   Logan Chien
   Catalin Gabriel Manciu
   Jacob Oscarson
@@ -302,7 +304,6 @@
   Laurens Van Houtven
   Bobby Impollonia
   Roberto De Ioris
-  Yannick Jadoul
   Jeong YunWon
   Christopher Armstrong
   Aaron Tubbs
@@ -357,6 +358,7 @@
   Daniil Yarancev
   Min RK
   OlivierBlanvillain
+  bernd.schoeller at inf.ethz.ch
   dakarpov at gmail.com
   Jonas Pfannschmidt
   Zearin
@@ -398,6 +400,7 @@
   Jesdi
   Konrad Delong
   Dinu Gherman
+  Sam Edwards
   pizi
   Tom&#225;&#353; Pru&#382;ina
   James Robert
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -73,7 +73,7 @@
 # The short X.Y version.
 version = '7.3'
 # The full version, including alpha/beta/rc tags.
-release = '7.3.0'
+release = '7.3.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -65,16 +65,16 @@
   Spenser Bauman
   Michal Bendowski
   Jan de Mooij
+  Stefano Rivera
   Tyler Wade
+  Stefan Beyer
   Vincent Legoll
   Michael Foord
   Stephan Diehl
-  Stefano Rivera
   Jean-Paul Calderone
   Stefan Schwarzer
   Tomek Meka
   Valentino Volonghi
-  Stefan Beyer
   Patrick Maupin
   Devin Jeanpierre
   Bob Ippolito
@@ -103,9 +103,10 @@
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
+  Stian Andreassen
+  Julian Berman
   William Leslie
   Pawe&#322; Piotr Przeradowski
-  Stian Andreassen
   marky1991
   Ilya Osadchiy
   Tobias Oberstein
@@ -116,7 +117,7 @@
   tav
   Georg Brandl
   Joannah Nanjekye
-  Julian Berman
+  Yannick Jadoul
   Bert Freudenberg
   Wanja Saatkamp
   Mike Blume
@@ -241,6 +242,7 @@
   Lutz Paelike
   Ian Foote
   Philipp Rustemeuer
+  Bernd Schoeller
   Logan Chien
   Catalin Gabriel Manciu
   Jacob Oscarson
@@ -253,7 +255,6 @@
   Lene Wagner
   Tomo Cocoa
   Miro Hron&#269;ok
-  Anthony Sottile
   David Lievens
   Neil Blakey-Milner
   Henrik Vendelbo
@@ -269,7 +270,6 @@
   Laurens Van Houtven
   Bobby Impollonia
   Roberto De Ioris
-  Yannick Jadoul
   Jeong YunWon
   Christopher Armstrong
   Aaron Tubbs
@@ -324,6 +324,7 @@
   Daniil Yarancev
   Min RK
   OlivierBlanvillain
+  bernd.schoeller at inf.ethz.ch
   dakarpov at gmail.com
   Jonas Pfannschmidt
   Zearin
@@ -365,6 +366,7 @@
   Jesdi
   Konrad Delong
   Dinu Gherman
+  Sam Edwards
   pizi
   Tom&#225;&#353; Pru&#382;ina
   James Robert
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-v7.3.0.rst
    release-v7.2.0.rst
    release-v7.1.1.rst
    release-v7.1.0.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-7.3.0.rst
    whatsnew-pypy2-7.2.0.rst
    whatsnew-pypy2-7.1.0.rst
    whatsnew-pypy2-7.0.0.rst
@@ -43,6 +44,7 @@
 
 .. toctree::
     whatsnew-pypy3-head.rst
+    whatsnew-pypy3-7.3.0.rst
     whatsnew-pypy3-7.2.0.rst
     whatsnew-pypy3-7.1.0.rst
 
diff --git a/pypy/doc/release-v7.3.0.rst b/pypy/doc/release-v7.3.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-v7.3.0.rst
@@ -0,0 +1,213 @@
+====================================
+PyPy v7.3.0: release of 2.7, and 3.6
+====================================
+
+The PyPy team is proud to release the version 7.3.0 of PyPy, which includes
+two different interpreters:
+
+  - PyPy2.7, which is an interpreter supporting the syntax and the features of
+    Python 2.7 including the stdlib for CPython 2.7.13
+
+  - PyPy3.6: which is an interpreter supporting the syntax and the features of
+    Python 3.6, including the stdlib for CPython 3.6.9.
+    
+The interpreters are based on much the same codebase, thus the double
+release.
+
+We have worked with the python packaging group to support tooling around
+building third party packages for python, so this release changes the ABI tag
+for PyPy.
+
+The `CFFI`_ backend has been updated to version 1.13.1. We recommend using CFFI
+rather than c-extensions to interact with C.
+
+The built-in ``_cppyy`` module was upgraded to 1.10.6, which
+provides, among others, better template resolution, stricter ``enum`` handling,
+anonymous struct/unions, cmake fragments for distribution, optimizations for
+PODs, and faster wrapper calls. We reccomend using cppyy_ for performant
+wrapping of C++ code for Python.
+
+The vendored pyrepl package for interaction inside the REPL was updated.
+
+Support for codepage encoding and decoding was added for Windows.
+
+As always, this release fixed several issues and bugs raised by the growing
+community of PyPy users.  We strongly recommend updating. Many of the fixes are
+the direct result of end-user bug reports, so please continue reporting issues
+as they crop up.
+
+You can download the v7.3 releases here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project. If PyPy is not quite good enough for your needs, we are available for
+direct consulting work.
+
+We would also like to thank our contributors and encourage new people to join
+the project. PyPy has many layers and we need help with all of them: `PyPy`_
+and `RPython`_ documentation improvements, tweaking popular modules to run
+on pypy, or general `help`_ with making RPython's JIT even better. Since the
+previous release, we have accepted contributions from 3 new contributors,
+thanks for pitching in.
+
+.. _`PyPy`: index.html
+.. _`RPython`: https://rpython.readthedocs.org
+.. _`help`: project-ideas.html
+.. _`CFFI`: http://cffi.readthedocs.io
+.. _`cppyy`: https://cppyy.readthedocs.io
+.. _`available as wheels`: https://github.com/antocuni/pypy-wheels
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7, 3.6. It's fast (`PyPy and CPython 2.7.x`_ performance
+comparison) due to its integrated tracing JIT compiler.
+
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
+
+This PyPy release supports:
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
+
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+  * 64-bit **ARM** machines running Linux.
+
+Unfortunately at the moment of writing our ARM buildbots are out of service,
+so for now we are **not** releasing any binary for the ARM architecture (32
+bit), although PyPy does support ARM 32 bit processors. 
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html
+
+
+Changelog
+=========
+
+Changes shared across versions
+------------------------------
+
+* Fix segfault when calling descr-methods with no arguments
+* Change the SOABI and subsequently change the reported ABI tag.
+* Update the builtin ``_cppyy`` backend to 1.10.6
+* Performance improvements in string/unicode handling
+* Fix compilation error when building `revdb`_ (`issue 3084`_, actually
+  released in PyPy 7.2 but not mentioned in the release notes)
+* Add JIT options to documentation and an option for JIT help via ``pypy --jit
+  help``
+* Fix regex escapes in pyrepl (`issue 3097`_)
+* Allow reloading of subclasses of module (`issue 3099`_)
+* Work around a gcc bug, which was reported to them and fixed (`issue 3086`_)
+* Fix (again) faulty logic when decoding invalid UTF-8 (`issue 2389`_)
+* Fix up LICENSE file
+* Turn all ``http`` links in the documentation to ``https``
+* Update the bndled pip and setuptools (used in ``pypy -mensurepip`` to version
+  that support `manylinux2010`_ wheels
+* Link the ``DEFAULT_SOABI`` to the ``PYPY_VERSION``
+* Workaround for programs calling ``sys.setrecursionlimit(huge_value)`` (`issue
+  3094`_)
+* Set minimal ``MACOSX_DEPLOYMENT_TARGET`` to 10.7 on macOS; cpython uses 10.5
+* Fix a JIT bug causing rare register corruption on aarch64
+* Add discovery of ``ncursesw`` when building ``_minimal_curses`` and improve
+  handling of old curses versions (`issue 2970`_)
+* Improve the error message for ``class X(random_object)`` (`issue 3109`_)
+* Deal with json dicts containing repeated keys in the new map based parser
+  (`issue 3108`_)
+* Port parts of the `portable pypy`_ repackaging scripts to add an option for
+  ``RPATH`` manipulation on linux
+* Check for overflow in ctypes array creation
+* Better support and report MSVC versions used to compile on windows
+* Allow any kind of buffer in socket.setsockopt(), like CPython (`issue 3114`_)
+
+C-API (cpyext) and c-extensions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* Add ``_PySet_Next``, ``_PySet_NextEntry``
+* Correctly swallow exceptions happening inside ``PyDict_GetItem()`` (`issue
+  3098`_)
+* Respect tp_dict on PyType_Ready (`issue XXX`_)
+* Allow calling ``PyType_Ready`` on a subclass with a partially built
+  ``tp_base`` (issue 3117`_)
+* Rename ``tuple_new`` to ``_PyPy_tuple_new`` to follow the naming convention of
+  exported symbols in ``libpypy-c.so``
+* Actually restore the traceback in ``PyErr_Restore`` (`issue 3120`_)
+
+Python 3.6 only
+---------------
+
+* Don't grow the ``lzma.decompress()`` buffer past ``max_length`` (`issue 3088`_)
+* Backport fix from CPython for failure of ``lzma`` to decompress a file
+  (`issue 3090`_)
+* Fix ``asyncgen_hooks`` and refactor ``coroutine execution``
+* Fix range checking in GB18030 decoder (CPython issue `29990`_)
+* Fix handling escape characters in HZ codec (CPython issue `30003`_)
+* Reject null characters in a few more functions (CPython issue `13617`_)
+* Fix build on macOS without ``clock_gettime`` (before 10.12 and xcode 8,
+  released 2016)
+* Backport 3.7.5 changes to ``timedelta.__eq__`` and ``time.__eq__`` (CPython
+  issue `37579`_)
+* Backport more fixes to comparisons in ``datetime.py`` (CPython issue `37985`_)
+* Use the python tag in ``pyc`` file names, not the abi tag
+* Handle string formatting with a single ``[`` in the format string (`issue
+  3100`_)
+* Backport some of the patches in `macports pypy`_
+* Add missing ``HAVE_FACCESSAT`` to ``posix._have_functions``
+* Update pyrepl from upstream package (`issue 2971`_)
+* Fix ``PyFrame._guess_function_name_parens()``
+* Fix range of allowed years in ``time.mktime`` to match CPython `13312`_
+* Generators need to store the old current ``exc_info`` in a place that is
+  visible, because in one corner case a call to ``sys.exc_info()`` might need
+  it.  (`issue 3096`_)
+* Remove incorrect clobbering of the ``locals`` after running ``exec()``
+* Adds encoding, decoding codepages on win32
+
+Python 3.6 C-API
+~~~~~~~~~~~~~~~~
+
+* Add ``PyObject_GenericGetDict``, ``PyObject_GenericSetDict``, ``_Py_strhex``,
+  ``_Py_strhex_bytes``, ``PyUnicodeNew``, ``_PyFinalizing``,
+  ``PySlice_Unpack``, ``PySlice_AdjustIndices``
+* Implement ``pystrhex.h`` (`issue 2687`_)
+* Make ``PyUnicodeObject`` slightly more compact
+* Fix memory leak when releasing a ``PyUnicodeObject``
+
+.. _`revdb`: fix broken link
+.. _`portable pypy`: fix broken link
+.. _`manylinux2010`: fix broken link
+.. _`macports pypy`: https://github.com/macports/macports-ports/blob/master/lang/pypy/files/darwin.py.diff
+
+.. _`issue 2389`: https://bitbucket.com/pypy/pypy/issues/2389
+.. _`issue 2687`: https://bitbucket.com/pypy/pypy/issues/2687
+.. _`issue 2970`: https://bitbucket.com/pypy/pypy/issues/2970
+.. _`issue 2971`: https://bitbucket.com/pypy/pypy/issues/2971
+.. _`issue 3084`: https://bitbucket.com/pypy/pypy/issues/3084
+.. _`issue 3086`: https://bitbucket.com/pypy/pypy/issues/3086
+.. _`issue 3088`: https://bitbucket.com/pypy/pypy/issues/3088
+.. _`issue 3090`: https://bitbucket.com/pypy/pypy/issues/3090
+.. _`issue 3094`: https://bitbucket.com/pypy/pypy/issues/3094
+.. _`issue 3096`: https://bitbucket.com/pypy/pypy/issues/3096
+.. _`issue 3097`: https://bitbucket.com/pypy/pypy/issues/3097
+.. _`issue 3098`: https://bitbucket.com/pypy/pypy/issues/3098
+.. _`issue 3099`: https://bitbucket.com/pypy/pypy/issues/3099
+.. _`issue 3100`: https://bitbucket.com/pypy/pypy/issues/3100
+.. _`issue 3108`: https://bitbucket.com/pypy/pypy/issues/3108
+.. _`issue 3109`: https://bitbucket.com/pypy/pypy/issues/3109
+.. _`issue 3114`: https://bitbucket.com/pypy/pypy/issues/3114
+.. _`issue 3117`: https://bitbucket.com/pypy/pypy/issues/3117
+.. _`issue 3120`: https://bitbucket.com/pypy/pypy/issues/3120
+
+.. _13312: https://bugs.python.org/issue13312
+.. _13617: https://bugs.python.org/issue13617
+.. _29990: https://bugs.python.org/issue29990
+.. _30003: https://bugs.python.org/issue30003
+.. _37579: https://bugs.python.org/issue37579
+.. _37985: https://bugs.python.org/issue37985
+.. _37985: https://bugs.python.org/issue37985
+
+
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -1,22 +1,8 @@
-==========================
-What's new in PyPy2.7 7.3+
-==========================
+============================
+What's new in PyPy2.7 7.3.0+
+============================
 
-.. this is a revision shortly after release-pypy-7.2.0
-.. startrev: a511d86377d6 
+.. this is a revision shortly after release-pypy-7.3.0
+.. startrev: dbbbae99135f 
 
-.. branch: fix-descrmismatch-crash
 
-Fix segfault when calling descr-methods with no arguments
-
-.. branch: https-readme
-
-Convert http -> https in README.rst
-
-.. branch: license-update
-
-Update list directories in LICENSE
-
-.. branch: allow-forcing-no-embed
-
-When packaging, allow suppressing embedded dependencies via PYPY_NO_EMBED_DEPENDENCIES
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-pypy2-7.3.0.rst
copy from pypy/doc/whatsnew-head.rst
copy to pypy/doc/whatsnew-pypy2-7.3.0.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-pypy2-7.3.0.rst
@@ -1,6 +1,6 @@
-==========================
-What's new in PyPy2.7 7.3+
-==========================
+===========================
+What's new in PyPy2.7 7.3.0
+===========================
 
 .. this is a revision shortly after release-pypy-7.2.0
 .. startrev: a511d86377d6 
@@ -19,4 +19,15 @@
 
 .. branch: allow-forcing-no-embed
 
-When packaging, allow suppressing embedded dependencies via PYPY_NO_EMBED_DEPENDENCIES
+When packaging, allow suppressing embedded dependencies via
+PYPY_NO_EMBED_DEPENDENCIES
+
+.. branch: int-test-is-zero
+
+.. branch: cppyy-dev
+
+Upgraded the built-in ``_cppyy`` module to ``cppyy-backend 1.10.6``, which
+provides, among others, better template resolution, stricter ``enum`` handling,
+anonymous struct/unions, cmake fragments for distribution, optimizations for
+PODs, and faster wrapper calls.
+
diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-7.3.0.rst
copy from pypy/doc/whatsnew-pypy3-head.rst
copy to pypy/doc/whatsnew-pypy3-7.3.0.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-7.3.0.rst
@@ -1,6 +1,6 @@
-========================
-What's new in PyPy3 7.2+
-========================
+=========================
+What's new in PyPy3 7.3.0
+=========================
 
 .. this is the revision after release-pypy3.6-v7.2
 .. startrev: 6d2f8470165b
diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -1,9 +1,9 @@
-========================
-What's new in PyPy3 7.2+
-========================
+==========================
+What's new in PyPy3 7.3.0+
+==========================
 
-.. this is the revision after release-pypy3.6-v7.2
-.. startrev: 6d2f8470165b
+.. this is the revision after release-pypy3.6-v7.3.0
+.. startrev: 78b4d0a7cf2e
 
 
 .. branch: py3.6-asyncgen
diff --git a/pypy/module/_cppyy/capi/loadable_capi.py b/pypy/module/_cppyy/capi/loadable_capi.py
--- a/pypy/module/_cppyy/capi/loadable_capi.py
+++ b/pypy/module/_cppyy/capi/loadable_capi.py
@@ -1,4 +1,4 @@
-import os
+import os, sys
 
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib.rarithmetic import intmask
@@ -20,17 +20,20 @@
 from pypy.module._cppyy.capi.capi_types import C_SCOPE, C_TYPE, C_OBJECT,\
    C_METHOD, C_INDEX, C_INDEX_ARRAY, WLAVC_INDEX, C_FUNC_PTR
 
-
-backend_library = 'libcppyy_backend.so'
+backend_ext = '.so'
+if 'win32' in sys.platform:
+    backend_ext = '.dll'
+backend_library = 'libcppyy_backend'
 
 # this is not technically correct, but will do for now
-std_string_name = 'std::basic_string<char>'
+std_string_name = 'std::string'
 
 class _Arg:         # poor man's union
     _immutable_ = True
-    def __init__(self, tc, h = 0, l = -1, d = -1., s = '', p = rffi.cast(rffi.VOIDP, 0)):
+    def __init__(self, tc, h = 0, u = 0, l = -1, d = -1., s = '', p = rffi.cast(rffi.VOIDP, 0)):
         self.tc      = tc
         self._handle = h
+        self._index  = u
         self._long   = l
         self._double = d
         self._string = s
@@ -41,6 +44,11 @@
     def __init__(self, val):
         _Arg.__init__(self, 'h', h = val)
 
+class _ArgU(_Arg):      # separate class for rtyper
+    _immutable_ = True
+    def __init__(self, val):
+        _Arg.__init__(self, 'u', u = val)
+
 class _ArgL(_Arg):
     _immutable_ = True
     def __init__(self, val):
@@ -95,7 +103,10 @@
                     misc.write_raw_signed_data(data, rffi.cast(rffi.LONG, obj._long), argtype.size)
                 elif obj.tc == 'h':
                     assert isinstance(argtype, ctypeprim.W_CTypePrimitiveUnsigned)
-                    misc.write_raw_unsigned_data(data, rffi.cast(rffi.ULONG, obj._handle), argtype.size)
+                    misc.write_raw_unsigned_data(data, rffi.cast(rffi.UINTPTR_T, obj._handle), argtype.size)
+                elif obj.tc == 'u':
+                    assert isinstance(argtype, ctypeprim.W_CTypePrimitiveUnsigned)
+                    misc.write_raw_unsigned_data(data, rffi.cast(rffi.SIZE_T, obj._index), argtype.size)
                 elif obj.tc == 'p':
                     assert obj._voidp != rffi.cast(rffi.VOIDP, 0)
                     data = rffi.cast(rffi.VOIDPP, data)
@@ -142,13 +153,17 @@
 
         # TODO: the following need to match up with the globally defined C_XYZ low-level
         # types (see capi/__init__.py), but by using strings here, that isn't guaranteed
-        c_opaque_ptr = state.c_ulong    # not ptrdiff_t (which is signed)
+        c_opaque_ptr = state.c_uintptr_t    # not size_t (which is signed)
+        c_size_t     = state.c_size_t
+        c_ptrdiff_t  = state.c_ptrdiff_t
+        c_intptr_t   = state.c_intptr_t
+        c_uintptr_t  = state.c_uintptr_t
  
         c_scope       = c_opaque_ptr
         c_type        = c_scope
-        c_object      = c_opaque_ptr    # not voidp (to stick with one handle type)
-        c_method      = c_opaque_ptr
-        c_index       = state.c_long
+        c_object      = c_opaque_ptr        # not voidp (to stick with one handle type)
+        c_method      = c_uintptr_t         # not intptr_t (which is signed)
+        c_index       = c_size_t
         c_index_array = state.c_voidp
 
         c_void    = state.c_void
@@ -166,10 +181,10 @@
         c_ccharp = state.c_ccharp
         c_voidp  = state.c_voidp
 
-        c_size_t = nt.new_primitive_type(space, 'size_t')
-        c_ptrdiff_t = nt.new_primitive_type(space, 'ptrdiff_t')
+        self.capi_call_ifaces = {
+            # direct interpreter access
+            'compile'                  : ([c_ccharp],                 c_int),
 
-        self.capi_call_ifaces = {
             # name to opaque C++ scope representation
             'resolve_name'             : ([c_ccharp],                 c_ccharp),
             'resolve_enum'             : ([c_ccharp],                 c_ccharp),
@@ -221,12 +236,17 @@
 
             'get_all_cpp_names'        : ([c_scope, c_voidp],         c_voidp), # const char**
 
+            # namespace reflection information
+            'get_using_namespaces'     : ([c_scope],                  c_index),
+
             # type/class reflection information
             'final_name'               : ([c_type],                   c_ccharp),
             'scoped_final_name'        : ([c_type],                   c_ccharp),
+            'has_virtual_destructor'   : ([c_type],                   c_int),
             'has_complex_hierarchy'    : ([c_type],                   c_int),
             'num_bases'                : ([c_type],                   c_int),
             'base_name'                : ([c_type, c_int],            c_ccharp),
+            'is_smartptr'              : ([c_type],                   c_int),
             'is_subtype'               : ([c_type, c_type],           c_int),
             'smartptr_info'            : ([c_ccharp, c_voidp, c_voidp],         c_int),
             'add_smartptr_type'        : ([c_ccharp],                 c_void),
@@ -245,9 +265,11 @@
             'method_result_type'       : ([c_method],                 c_ccharp),
             'method_num_args'          : ([c_method],                 c_int),
             'method_req_args'          : ([c_method],                 c_int),
+            'method_arg_name'          : ([c_method, c_int],          c_ccharp),
             'method_arg_type'          : ([c_method, c_int],          c_ccharp),
             'method_arg_default'       : ([c_method, c_int],          c_ccharp),
             'method_signature'         : ([c_method, c_int],          c_ccharp),
+            'method_signature_max'     : ([c_method, c_int, c_int],   c_ccharp),
             'method_prototype'         : ([c_scope, c_method, c_int], c_ccharp),
             'is_const_method'          : ([c_method],                 c_int),
 
@@ -269,7 +291,7 @@
             'num_datamembers'          : ([c_scope],                  c_int),
             'datamember_name'          : ([c_scope, c_int],           c_ccharp),
             'datamember_type'          : ([c_scope, c_int],           c_ccharp),
-            'datamember_offset'        : ([c_scope, c_int],           c_ptrdiff_t),
+            'datamember_offset'        : ([c_scope, c_int],           c_intptr_t),
             'datamember_index'         : ([c_scope, c_ccharp],        c_int),
 
             # data member properties
@@ -311,7 +333,15 @@
             state.backend = W_Library(space, space.newtext(libname), dldflags)
         else:
             # try usual lookups
-            state.backend = W_Library(space, space.newtext(backend_library), dldflags)
+            try:
+                if backend_library[-len(backend_ext):] == backend_ext:
+                    fullname = backend_library
+                else:
+                    fullname = backend_library+backend_ext
+                state.backend = W_Library(space, space.newtext(fullname), dldflags)
+            except Exception as e:
+                # TODO: where to find the value '.pypy-41'? Note that this only matters for testing.
+                state.backend = W_Library(space, space.newtext(backend_library+'.pypy-41'+backend_ext), dldflags)
 
         if state.backend:
             # fix constants
@@ -354,7 +384,10 @@
     return rffi.cast(rffi.SIZE_T, space.uint_w(w_cdata))
 
 def _cdata_to_ptrdiff_t(space, w_cdata):
-    return rffi.cast(rffi.LONG, space.int_w(w_cdata))
+    return rffi.cast(rffi.PTRDIFF_T, space.int_w(w_cdata))
+
+def _cdata_to_intptr_t(space, w_cdata):
+    return rffi.cast(rffi.INTPTR_T, space.int_w(w_cdata))
 
 def _cdata_to_ptr(space, w_cdata): # TODO: this is both a hack and dreadfully slow
     w_cdata = space.interp_w(cdataobj.W_CData, w_cdata, can_be_None=False)
@@ -365,6 +398,10 @@
     ptr = _cdata_to_ptr(space, w_cdata)      # see above ... something better?
     return rffi.cast(rffi.CCHARP, ptr)
 
+# direct interpreter access
+def c_compile(space, code):
+    return space.int_w(call_capi(space, 'compile', [_ArgS(code)]))
+
 # name to opaque C++ scope representation ------------------------------------
 def c_resolve_name(space, name):
     return charp2str_free(space, call_capi(space, 'resolve_name', [_ArgS(name)]))
@@ -488,11 +525,17 @@
     c_free(space, rffi.cast(rffi.VOIDP, rawnames))        # id.
     return allnames
 
+# namespace reflection information
+def c_get_using_namespaces(space, cppscope):
+    return space.uint_w(call_capi(space, 'get_using_namespaces', [_ArgH(cppscope)]))
+
 # type/class reflection information ------------------------------------------
 def c_final_name(space, cpptype):
     return charp2str_free(space, call_capi(space, 'final_name', [_ArgH(cpptype)]))
 def c_scoped_final_name(space, cpptype):
     return charp2str_free(space, call_capi(space, 'scoped_final_name', [_ArgH(cpptype)]))
+def c_has_virtual_destructor(space, handle):
+    return space.bool_w(call_capi(space, 'has_virtual_destructor', [_ArgH(handle)]))
 def c_has_complex_hierarchy(space, handle):
     return space.bool_w(call_capi(space, 'has_complex_hierarchy', [_ArgH(handle)]))
 def c_num_bases(space, cppclass):
@@ -500,6 +543,8 @@
 def c_base_name(space, cppclass, base_index):
     args = [_ArgH(cppclass.handle), _ArgL(base_index)]
     return charp2str_free(space, call_capi(space, 'base_name', args))
+def c_is_smartptr(space, handle):
+    return space.bool_w(call_capi(space, 'is_smartptr', [_ArgH(handle)]))
 def c_is_subtype(space, derived, base):
     jit.promote(base)
     if derived == base:
@@ -552,7 +597,7 @@
     return py_indices
 
 def c_get_method(space, cppscope, index):
-    args = [_ArgH(cppscope.handle), _ArgL(index)]
+    args = [_ArgH(cppscope.handle), _ArgU(index)]
     return rffi.cast(C_METHOD, space.uint_w(call_capi(space, 'get_method', args)))
 
 def c_method_name(space, cppmeth):
@@ -567,6 +612,9 @@
     return space.int_w(call_capi(space, 'method_num_args', [_ArgH(cppmeth)]))
 def c_method_req_args(space, cppmeth):
     return space.int_w(call_capi(space, 'method_req_args', [_ArgH(cppmeth)]))
+def c_method_arg_name(space, cppmeth, arg_index):
+    args = [_ArgH(cppmeth), _ArgL(arg_index)]
+    return charp2str_free(space, call_capi(space, 'method_arg_name', args))
 def c_method_arg_type(space, cppmeth, arg_index):
     args = [_ArgH(cppmeth), _ArgL(arg_index)]
     return charp2str_free(space, call_capi(space, 'method_arg_type', args))
@@ -576,6 +624,9 @@
 def c_method_signature(space, cppmeth, show_formalargs=True):
     args = [_ArgH(cppmeth), _ArgL(show_formalargs)]
     return charp2str_free(space, call_capi(space, 'method_signature', args))
+def c_method_signature_max(space, cppmeth, show_formalargs, maxargs):
+    args = [_ArgH(cppmeth), _ArgL(show_formalargs), _ArgL(maxargs)]
+    return charp2str_free(space, call_capi(space, 'method_signature_max', args))
 def c_method_prototype(space, cppscope, cppmeth, show_formalargs=True):
     args = [_ArgH(cppscope.handle), _ArgH(cppmeth), _ArgL(show_formalargs)]
     return charp2str_free(space, call_capi(space, 'method_prototype', args))
@@ -583,24 +634,24 @@
     return space.bool_w(call_capi(space, 'is_const_method', [_ArgH(cppmeth)]))
 
 def c_get_num_templated_methods(space, cppscope):
-    return space.int_w(call_capi(space, 'method_is_template', [_ArgH(cppscope.handle)]))
+    return space.int_w(call_capi(space, 'get_num_templated_methods', [_ArgH(cppscope.handle)]))
 def c_get_templated_method_name(space, cppscope, index):
-    args = [_ArgH(cppscope.handle), _ArgL(index)]
-    return charp2str_free(space, call_capi(space, 'method_is_template', args))
+    args = [_ArgH(cppscope.handle), _ArgU(index)]
+    return charp2str_free(space, call_capi(space, 'get_templated_method_name', args))
 def c_exists_method_template(space, cppscope, name):
     args = [_ArgH(cppscope.handle), _ArgS(name)]
     return space.bool_w(call_capi(space, 'exists_method_template', args))
 def c_method_is_template(space, cppscope, index):
-    args = [_ArgH(cppscope.handle), _ArgL(index)]
+    args = [_ArgH(cppscope.handle), _ArgU(index)]
     return space.bool_w(call_capi(space, 'method_is_template', args))
 def c_get_method_template(space, cppscope, name, proto):
     args = [_ArgH(cppscope.handle), _ArgS(name), _ArgS(proto)]
-    return rffi.cast(C_METHOD, space.uint_w(call_capi(space, 'get_method_template', args)))
+    return rffi.cast(C_METHOD, space.int_w(call_capi(space, 'get_method_template', args)))
 
 def c_get_global_operator(space, nss, lc, rc, op):
     if nss is not None:
         args = [_ArgH(nss.handle), _ArgH(lc.handle), _ArgH(rc.handle), _ArgS(op)]
-        return rffi.cast(WLAVC_INDEX, space.int_w(call_capi(space, 'get_global_operator', args)))
+        return rffi.cast(WLAVC_INDEX, space.uint_w(call_capi(space, 'get_global_operator', args)))
     return rffi.cast(WLAVC_INDEX, -1)
 
 # method properties ----------------------------------------------------------
@@ -624,7 +675,7 @@
     return  charp2str_free(space, call_capi(space, 'datamember_type', args))
 def c_datamember_offset(space, cppscope, datamember_index):
     args = [_ArgH(cppscope.handle), _ArgL(datamember_index)]
-    return _cdata_to_ptrdiff_t(space, call_capi(space, 'datamember_offset', args))
+    return _cdata_to_intptr_t(space, call_capi(space, 'datamember_offset', args))
 
 def c_datamember_index(space, cppscope, name):
     args = [_ArgH(cppscope.handle), _ArgS(name)]
diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py
--- a/pypy/module/_cppyy/converter.py
+++ b/pypy/module/_cppyy/converter.py
@@ -993,7 +993,7 @@
 _converters["void*&"]                   = VoidPtrRefConverter
 
 # special cases (note: 'string' aliases added below)
-_converters["std::basic_string<char>"]           = StdStringConverter
+_converters["std::string"]                       = StdStringConverter
 _converters["const std::basic_string<char>&"]    = StdStringConverter     # TODO: shouldn't copy
 _converters["std::basic_string<char>&"]          = StdStringRefConverter
 _converters["std::basic_string<char>&&"]         = StdStringMoveConverter
@@ -1126,7 +1126,8 @@
         ("char",                            "signed char"),   # TODO: check
         ("const char*",                     "char*"),
 
-        ("std::basic_string<char>",         "string"),
+        ("std::string",                     "string"),
+        ("std::string",                     "std::basic_string<char>"),
         ("const std::basic_string<char>&",  "const string&"),
         ("std::basic_string<char>&",        "string&"),
         ("std::basic_string<char>&&",       "string&&"),
diff --git a/pypy/module/_cppyy/executor.py b/pypy/module/_cppyy/executor.py
--- a/pypy/module/_cppyy/executor.py
+++ b/pypy/module/_cppyy/executor.py
@@ -388,7 +388,7 @@
 # special cases (note: 'string' aliases added below)
 _executors["constructor"]         = ConstructorExecutor
 
-_executors["std::basic_string<char>"]         = StdStringExecutor
+_executors["std::string"]         = StdStringExecutor
 _executors["const std::basic_string<char>&"]  = StdStringRefExecutor
 _executors["std::basic_string<char>&"]        = StdStringRefExecutor
 
@@ -457,7 +457,8 @@
     aliases = (
         ("const char*",                     "char*"),
 
-        ("std::basic_string<char>",         "string"),
+        ("std::string",                     "string"),
+        ("std::string",                     "std::basic_string<char>"),
         ("const std::basic_string<char>&",  "const string&"),
         ("std::basic_string<char>&",        "string&"),
 
diff --git a/pypy/module/_cppyy/ffitypes.py b/pypy/module/_cppyy/ffitypes.py
--- a/pypy/module/_cppyy/ffitypes.py
+++ b/pypy/module/_cppyy/ffitypes.py
@@ -38,6 +38,8 @@
         # special types
         self.c_size_t    = nt.new_primitive_type(space, 'size_t')
         self.c_ptrdiff_t = nt.new_primitive_type(space, 'ptrdiff_t')
+        self.c_intptr_t  = nt.new_primitive_type(space, 'intptr_t')
+        self.c_uintptr_t = nt.new_primitive_type(space, 'uintptr_t')
 
 class BoolTypeMixin(object):
     _mixin_     = True
diff --git a/pypy/module/_cppyy/include/capi.h b/pypy/module/_cppyy/include/capi.h
--- a/pypy/module/_cppyy/include/capi.h
+++ b/pypy/module/_cppyy/include/capi.h
@@ -2,22 +2,26 @@
 #define CPPYY_CAPI
 
 #include <stddef.h>
+#include <stdint.h>
 #include "src/precommondefs.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif // ifdef __cplusplus
 
-    typedef ptrdiff_t     cppyy_scope_t;
+    typedef size_t        cppyy_scope_t;
     typedef cppyy_scope_t cppyy_type_t;
     typedef void*         cppyy_object_t;
-    typedef ptrdiff_t     cppyy_method_t;
+    typedef intptr_t      cppyy_method_t;
 
-    typedef long          cppyy_index_t;
+    typedef size_t        cppyy_index_t;
     typedef void*         cppyy_funcaddr_t;
 
     typedef unsigned long cppyy_exctype_t;
 
+    /* direct interpreter access ---------------------------------------------- */
+    int cppyy_compile(const char* code);
+
     /* name to opaque C++ scope representation -------------------------------- */
     RPY_EXTERN
     char* cppyy_resolve_name(const char* cppitem_name);
@@ -103,12 +107,17 @@
     RPY_EXTERN
     const char** cppyy_get_all_cpp_names(cppyy_scope_t scope, size_t* count);
 
+    /* namespace reflection information --------------------------------------- */
+    cppyy_index_t* cppyy_get_using_namespaces(cppyy_scope_t scope);
+
     /* class reflection information ------------------------------------------- */
     RPY_EXTERN
     char* cppyy_final_name(cppyy_type_t type);
     RPY_EXTERN
     char* cppyy_scoped_final_name(cppyy_type_t type);
     RPY_EXTERN
+    int cppyy_has_virtual_destructor(cppyy_type_t type);
+    RPY_EXTERN
     int cppyy_has_complex_hierarchy(cppyy_type_t type);
     RPY_EXTERN
     int cppyy_num_bases(cppyy_type_t type);
@@ -117,6 +126,8 @@
     RPY_EXTERN
     int cppyy_is_subtype(cppyy_type_t derived, cppyy_type_t base);
     RPY_EXTERN
+    int cppyy_is_smartptr(cppyy_type_t type);
+    RPY_EXTERN
     int cppyy_smartptr_info(const char* name, cppyy_type_t* raw, cppyy_method_t* deref);
     RPY_EXTERN
     void cppyy_add_smartptr_type(const char* type_name);
@@ -147,20 +158,24 @@
     RPY_EXTERN
     int cppyy_method_req_args(cppyy_method_t);
     RPY_EXTERN
+    char* cppyy_method_arg_name(cppyy_method_t,int arg_index);
+    RPY_EXTERN
     char* cppyy_method_arg_type(cppyy_method_t, int arg_index);
     RPY_EXTERN
     char* cppyy_method_arg_default(cppyy_method_t, int arg_index);
     RPY_EXTERN
     char* cppyy_method_signature(cppyy_method_t, int show_formalargs);
     RPY_EXTERN
+    char* cppyy_method_signature_max(cppyy_method_t, int show_formalargs, int maxargs);
+    RPY_EXTERN
     char* cppyy_method_prototype(cppyy_scope_t scope, cppyy_method_t, int show_formalargs);
     RPY_EXTERN
     int cppyy_is_const_method(cppyy_method_t);
 
     RPY_EXTERN
-    int get_num_templated_methods(cppyy_scope_t scope);
+    int cppyy_get_num_templated_methods(cppyy_scope_t scope);
     RPY_EXTERN
-    char* get_templated_method_name(cppyy_scope_t scope, cppyy_index_t imeth);
+    char* cppyy_get_templated_method_name(cppyy_scope_t scope, cppyy_index_t imeth);
     RPY_EXTERN
     int cppyy_exists_method_template(cppyy_scope_t scope, const char* name);
     RPY_EXTERN
@@ -190,7 +205,7 @@
     RPY_EXTERN
     char* cppyy_datamember_type(cppyy_scope_t scope, int datamember_index);
     RPY_EXTERN
-    ptrdiff_t cppyy_datamember_offset(cppyy_scope_t scope, int datamember_index);
+    intptr_t cppyy_datamember_offset(cppyy_scope_t scope, int datamember_index);
     RPY_EXTERN
     int cppyy_datamember_index(cppyy_scope_t scope, const char* name);
 
diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -978,8 +978,8 @@
 )
 
 class W_CPPTemplateStaticOverload(W_CPPStaticOverload, TemplateOverloadMixin):
-    """App-level dispatcher to allow both lookup/instantiation of templated methods and
-    dispatch among overloads between templated and non-templated method."""
+    """Dispatcher to allow both lookup/instantiation of templated methods and
+    select among templated and non-templated method overloads."""
 
     _attrs_ = ['name', 'tmpl_args', 'overloads', 'master', 'w_this']
     _immutable_fields_ = ['name', 'tmpl_args']
@@ -993,7 +993,8 @@
          self.w_this = space.w_None
 
     def clone(self, tmpl_args):
-        other = W_CPPTemplateStaticOverload(self.space, self.name, tmpl_args, self.scope, self.functions, self.flags)
+        other = W_CPPTemplateStaticOverload(self.space, self.name,
+            tmpl_args, self.scope, self.functions, self.flags)
         other.overloads = self.overloads
         other.master = self.master
         other.w_this = self.w_this
@@ -1004,7 +1005,8 @@
         if isinstance(w_cppinstance, W_CPPInstance):
             cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance)
             if cppinstance.clsdecl.handle != self.scope.handle:
-                cppol = W_CPPTemplateStaticOverload(self.space, self.name, self.tmpl_args, self.scope, self.functions, self.flags)
+                cppol = W_CPPTemplateStaticOverload(self.space, self.name,
+                    self.tmpl_args, self.scope, self.functions, self.flags)
                 cppol.w_this = w_cppinstance
                 cppol.master = self.master
                 return cppol       # bound
@@ -1253,10 +1255,10 @@
         return self.handle != other.handle
 
 
-# For now, keep namespaces and classes separate as namespaces are extensible
+# Namespaces and classes are separate as namespaces are (more) extensible
 # with info from multiple dictionaries and do not need to bother with meta
-# classes for inheritance. Both are python classes, though, and refactoring
-# may be in order at some point.
+# classes for inheritance. Both are python classes, though, and further
+# refactoring may be in order at some point.
 class W_CPPNamespaceDecl(W_CPPScopeDecl):
     _attrs_ = ['space', 'handle', 'name', 'overloads', 'datamembers']
     _immutable_fields_ = ['handle', 'name']
@@ -1352,6 +1354,8 @@
     def _build_overloads(self):
         assert len(self.overloads) == 0
         methods_tmp = {}; ftype_tmp = {}
+
+        # add all ordinary methods (incl. pre-instantiated templates)
         for idx in range(capi.c_num_methods(self.space, self)):
             cppmeth = capi.c_get_method(self.space, self, idx)
             if capi.c_is_constructor(self.space, cppmeth):
@@ -1369,6 +1373,7 @@
             ftype_tmp[pyname] |= self._make_cppfunction(pyname, cppmeth, methods)
             if capi.c_method_is_template(self.space, self, idx):
                 ftype_tmp[pyname] |= FUNCTION_IS_TEMPLATE
+
         # the following covers the case where the only kind of operator[](idx)
         # returns are the ones that produce non-const references; these can be
         # used for __getitem__ just as much as for __setitem__, though
@@ -1404,6 +1409,11 @@
                 overload = W_CPPOverload(self.space, self, methods[:])
             self.overloads[pyname] = overload
 
+        # add placeholders for all non-instantiated templated methods
+        for idx in range(capi.c_get_num_templated_methods(self.space, self)):
+            cppname = capi.c_get_templated_method_name(self.space, self, idx)
+            self.overloads[cppname] = W_CPPTemplateOverload(self.space, cppname, None, self, [])
+
     def _make_cppfunction(self, pyname, cppmeth, funcs):
         num_args = capi.c_method_num_args(self.space, cppmeth)
         args_required = capi.c_method_req_args(self.space, cppmeth)
diff --git a/pypy/module/_cppyy/pythonify.py b/pypy/module/_cppyy/pythonify.py
--- a/pypy/module/_cppyy/pythonify.py
+++ b/pypy/module/_cppyy/pythonify.py
@@ -439,14 +439,17 @@
     # also the fallback on the indexed __getitem__, but that is slower)
     add_checked_item = False
     if name.find('std::vector', 0, 11) != 0:
-        if ('begin' in pyclass.__dict__ and 'end' in pyclass.__dict__):
+        if 'begin' in pyclass.__dict__ and 'end' in pyclass.__dict__:
             if _cppyy._scope_byname(name+'::iterator') or \
                     _cppyy._scope_byname(name+'::const_iterator'):
                 def __iter__(self):
                     i = self.begin()
-                    while i != self.end():
+                    end = self.size()
+                    count = 0
+                    while count != end:
                         yield i.__deref__()
                         i.__preinc__()
+                        count += 1
                     i.__destruct__()
                     raise StopIteration
                 pyclass.__iter__ = __iter__
diff --git a/pypy/module/_cppyy/src/dummy_backend.cxx b/pypy/module/_cppyy/src/dummy_backend.cxx
--- a/pypy/module/_cppyy/src/dummy_backend.cxx
+++ b/pypy/module/_cppyy/src/dummy_backend.cxx
@@ -160,7 +160,7 @@
 struct Cppyy_InitPseudoReflectionInfo {
     Cppyy_InitPseudoReflectionInfo() {
         // class example01 --
-        static long s_scope_id  = 0;
+        static cppyy_scope_t s_scope_id  = 0;
 
         { // namespace ''
         s_handles[""] = (cppyy_scope_t)++s_scope_id;
@@ -450,14 +450,14 @@
     Cppyy_PseudoMethodInfo* idx = (Cppyy_PseudoMethodInfo*)method;
     if (idx == s_methods["static_example01::staticSetPayload_payload*_double"]) {
         assert(!self && nargs == 2);
-        dummy::example01::staticSetPayload((dummy::payload*)(*(long*)&((CPPYY_G__value*)args)[0]),
+        dummy::example01::staticSetPayload((dummy::payload*)(*(intptr_t*)&((CPPYY_G__value*)args)[0]),
            ((CPPYY_G__value*)args)[1].obj.d);
     } else if (idx == s_methods["static_example01::setCount_int"]) {
         assert(!self && nargs == 1);
         dummy::example01::setCount(((CPPYY_G__value*)args)[0].obj.in);
     } else if (idx == s_methods["example01::setPayload_payload*"]) {
         assert(self && nargs == 1);
-        ((dummy::example01*)self)->setPayload((dummy::payload*)(*(long*)&((CPPYY_G__value*)args)[0]));
+        ((dummy::example01*)self)->setPayload((dummy::payload*)(*(intptr_t*)&((CPPYY_G__value*)args)[0]));
     } else if (idx == s_methods["CppyyTestData::destroy_arrays"]) {
         assert(self && nargs == 0);
         ((dummy::CppyyTestData*)self)->destroy_arrays();
@@ -591,11 +591,11 @@
         result = dummy::example01::staticAddOneToInt(((CPPYY_G__value*)args)[0].obj.in);
     } else if (idx == s_methods["static_example01::staticAddOneToInt_int_int"]) {
         assert(!self && nargs == 2);
-        result =  dummy::example01::staticAddOneToInt(
+        result = dummy::example01::staticAddOneToInt(
            ((CPPYY_G__value*)args)[0].obj.in, ((CPPYY_G__value*)args)[1].obj.in);
     } else if (idx == s_methods["static_example01::staticAtoi_cchar*"]) {
         assert(!self && nargs == 1);
-        result = dummy::example01::staticAtoi((const char*)(*(long*)&((CPPYY_G__value*)args)[0]));
+        result = dummy::example01::staticAtoi((const char*)(*(intptr_t*)&((CPPYY_G__value*)args)[0]));
     } else if (idx == s_methods["static_example01::getCount"]) {
         assert(!self && nargs == 0);
         result = dummy::example01::getCount();
@@ -605,7 +605,7 @@
     } else if (idx == s_methods["example01::addDataToAtoi_cchar*"]) {
         assert(self && nargs == 1);
         result = ((dummy::example01*)self)->addDataToAtoi(
-           (const char*)(*(long*)&((CPPYY_G__value*)args)[0]));
+           (const char*)(*(intptr_t*)&((CPPYY_G__value*)args)[0]));
     } else if (idx == s_methods["CppyyTestData::get_int"]) {
         assert(self && nargs == 0);
         result = ((dummy::CppyyTestData*)self)->get_int();
@@ -621,20 +621,20 @@
     if (idx == s_methods["static_example01::staticStrcpy_cchar*"]) {
         assert(!self && nargs == 1);
         result = (long)dummy::example01::staticStrcpy(
-           (const char*)(*(long*)&((CPPYY_G__value*)args)[0]));
+           (const char*)(*(intptr_t*)&((CPPYY_G__value*)args)[0]));
     } else if (idx == s_methods["static_example01::staticCyclePayload_payload*_double"]) {
         assert(!self && nargs == 2);
         result = (long)dummy::example01::staticCyclePayload(
-           (dummy::payload*)(*(long*)&((CPPYY_G__value*)args)[0]),
+           (dummy::payload*)(*(intptr_t*)&((CPPYY_G__value*)args)[0]),
            ((CPPYY_G__value*)args)[1].obj.d);
     } else if (idx == s_methods["example01::addToStringValue_cchar*"]) {
         assert(self && nargs == 1);
         result = (long)((dummy::example01*)self)->addToStringValue(
-           (const char*)(*(long*)&((CPPYY_G__value*)args)[0]));
+           (const char*)(*(intptr_t*)&((CPPYY_G__value*)args)[0]));
     } else if (idx == s_methods["example01::cyclePayload_payload*"]) {
         assert(self && nargs == 1);
         result = (long)((dummy::example01*)self)->cyclePayload(
-           (dummy::payload*)(*(long*)&((CPPYY_G__value*)args)[0]));
+           (dummy::payload*)(*(intptr_t*)&((CPPYY_G__value*)args)[0]));
     } else if (idx == s_methods["CppyyTestData::get_uint"]) {
         assert(self && nargs == 0);
         result = (long)((dummy::CppyyTestData*)self)->get_uint();
@@ -731,7 +731,7 @@
     } else if (idx == s_methods["CppyyTestData::pass_void_array_l"]) {
         assert(self && nargs == 1);
         result = (long)((dummy::CppyyTestData*)self)->pass_void_array_l(
-           (*(long**)&((CPPYY_G__value*)args)[0]));
+           (*(intptr_t**)&((CPPYY_G__value*)args)[0]));
     } else if (idx == s_methods["CppyyTestData::pass_array_ulong"]) {
         assert(self && nargs == 1);
         result = (long)((dummy::CppyyTestData*)self)->pass_array(
@@ -739,7 +739,7 @@
     } else if (idx == s_methods["CppyyTestData::pass_void_array_L"]) {
         assert(self && nargs == 1);
         result = (long)((dummy::CppyyTestData*)self)->pass_void_array_L(
-           (*(unsigned long**)&((CPPYY_G__value*)args)[0]));
+           (*(uintptr_t**)&((CPPYY_G__value*)args)[0]));
     } else if (idx == s_methods["CppyyTestData::pass_array_float"]) {
         assert(self && nargs == 1);
         result = (long)((dummy::CppyyTestData*)self)->pass_array(
@@ -868,7 +868,7 @@
     Cppyy_PseudoMethodInfo* idx = (Cppyy_PseudoMethodInfo*)method;
     if (idx == s_methods["static_example01::staticStrcpy_cchar*"]) {
         assert(!self && nargs == 1);
-        result = dummy::example01::staticStrcpy((const char*)(*(long*)&((CPPYY_G__value*)args)[0]));
+        result = dummy::example01::staticStrcpy((const char*)(*(intptr_t*)&((CPPYY_G__value*)args)[0]));
     } else {
         assert(!"method unknown in cppyy_call_s");
     }
@@ -903,7 +903,8 @@
 
 /* handling of function argument buffer ----------------------------------- */
 void* cppyy_allocate_function_args(int nargs) {
-    CPPYY_G__value* args = (CPPYY_G__value*)malloc(nargs*sizeof(CPPYY_G__value));
+    /* nargs parameters + one unsigned long for exception status output */
+    CPPYY_G__value* args = (CPPYY_G__value*)malloc(nargs*sizeof(CPPYY_G__value)+sizeof(unsigned long));
     for (int i = 0; i < nargs; ++i)
         args[i].type = 'l';
     return (void*)args;
@@ -1023,6 +1024,10 @@
     return cppstring_to_cstring("");
 }
 
+int cppyy_get_num_templated_methods(cppyy_scope_t scope) {
+    return 0;
+}
+
 int cppyy_exists_method_template(cppyy_scope_t scope, const char* name) {
     return 0;
 }
diff --git a/pypy/module/_cppyy/test/Makefile b/pypy/module/_cppyy/test/Makefile
--- a/pypy/module/_cppyy/test/Makefile
+++ b/pypy/module/_cppyy/test/Makefile
@@ -15,7 +15,7 @@
 
 HASGENREFLEX:=$(shell command -v genreflex 2> /dev/null)
 
-cppflags=-std=c++14 -O3 -fPIC -rdynamic
+cppflags=$(shell cling-config --cppflags) -O3 -fPIC -rdynamic
 ifdef HASGENREFLEX
   genreflex_flags:=$(shell genreflex --cppflags)
   cppflags+=$(genreflex_flags)
diff --git a/pypy/module/_cppyy/test/templates.cxx b/pypy/module/_cppyy/test/templates.cxx
--- a/pypy/module/_cppyy/test/templates.cxx
+++ b/pypy/module/_cppyy/test/templates.cxx
@@ -10,3 +10,28 @@
 long MyTemplatedMethodClass::get_float_size()  { return (long)sizeof(float); }
 long MyTemplatedMethodClass::get_double_size() { return (long)sizeof(double); }
 long MyTemplatedMethodClass::get_self_size()   { return (long)sizeof(MyTemplatedMethodClass); }
+
+
+// variadic templates
+#ifdef WIN32
+__declspec(dllexport)
+#endif
+std::string some_variadic::gTypeName = "";
+
+
+// template with empty body
+namespace T_WithEmptyBody {
+
+#ifdef WIN32
+__declspec(dllexport)
+#endif
+std::string side_effect = "not set";
+
+template<typename T>
+void some_empty() {
+    side_effect = "side effect";
+}
+
+template void some_empty<int>();
+
+} // namespace T_WithRValue
diff --git a/pypy/module/_cppyy/test/templates.h b/pypy/module/_cppyy/test/templates.h
--- a/pypy/module/_cppyy/test/templates.h
+++ b/pypy/module/_cppyy/test/templates.h
@@ -1,12 +1,33 @@
+#ifndef CPPYY_TEST_TEMPLATES_H
+#define CPPYY_TEST_TEMPLATES_H
+
+#include <stdexcept>
 #include <string>
 #include <sstream>
 #include <vector>
 
+#ifndef WIN32
+#include <cxxabi.h>
+inline std::string demangle_it(const char* name, const char* errmsg) {
+    int status;
+    std::string res = abi::__cxa_demangle(name, 0, 0, &status);
+    if (status != 0) throw std::runtime_error(errmsg);
+    return res;
+}
+#else
+inline std::string demangle_it(const char* name, const char*) {
+    return name;        // typeinfo's name() is already demangled
+}
+#endif
+
 
 //===========================================================================
 class MyTemplatedMethodClass {         // template methods
 public:
-    long get_size();      // to get around bug in genreflex
+    template<class A> long get_size(A&);
+    template<class A> long get_size(const A&);
+
+    long get_size();
     template<class B> long get_size();
 
     long get_char_size();
@@ -21,6 +42,16 @@
     double m_data[3];
 };
 
+template<class A>
+long MyTemplatedMethodClass::get_size(A&) {
+    return sizeof(A);
+}
+
+template<class A>
+long MyTemplatedMethodClass::get_size(const A&) {
+    return sizeof(A)+1;
+}
+
 template<class B>
 inline long MyTemplatedMethodClass::get_size() {
     return sizeof(B);
@@ -63,7 +94,7 @@
 };
 
 template <class I, typename O = float>
-SomeResult<O> global_get_some_result(const std::vector<I>& carrier) {
+SomeResult<O> global_get_some_result(const I& carrier) {
     SomeResult<O> r{};
     r.m_retval = O(carrier[0]);
     return r;
@@ -156,3 +187,295 @@
     T m_value;
 };
 */
+
+
+//===========================================================================
+// templated callable
+class TemplatedCallable {
+public:
+    template <class I , class O = double>
+    O operator() (const I& in) const { return O(in); }
+};
+
+
+//===========================================================================
+// templated typedefs
+namespace TemplatedTypedefs {
+
+template<typename TYPE_IN, typename TYPE_OUT, size_t _vsize = 4>
+struct BaseWithEnumAndTypedefs {
+    enum { vsize = _vsize };
+    typedef TYPE_IN in_type;
+    typedef TYPE_OUT out_type;
+};
+
+template <typename TYPE_IN, typename TYPE_OUT, size_t _vsize = 4>
+struct DerivedWithUsing : public BaseWithEnumAndTypedefs<TYPE_IN, TYPE_OUT, _vsize>
+{
+    typedef BaseWithEnumAndTypedefs<TYPE_IN, TYPE_OUT, _vsize> base_type;
+    using base_type::vsize;
+    using typename base_type::in_type;
+    typedef typename base_type::in_type in_type_tt;
+    using typename base_type::out_type;
+};
+
+struct SomeDummy {};
+
+} // namespace TemplatedTypedefs
+
+
+//===========================================================================
+// hiding templated methods
+namespace TemplateHiding {
+
+struct Base {
+    template<class T>
+    int callme(T t = T(1)) { return 2*t; }
+};
+
+struct Derived : public Base {
+    int callme(int t = 2) { return t; }
+};
+
+} // namespace TemplateHiding
+
+
+//===========================================================================
+// 'using' of templates
+template<typename T> using DA_vector = std::vector<T>;
+
+#if __cplusplus > 201402L
+namespace using_problem {
+
+template <typename T, size_t SZ>
+struct vector {
+    vector() : m_val(SZ) {}
+    T m_val;
+};
+
+template <typename T, size_t ... sizes>
+struct matryoshka {
+    typedef T type;
+};
+
+template <typename T, size_t SZ, size_t ... sizes>
+struct matryoshka<T, SZ, sizes ... > {
+    typedef vector<typename matryoshka<T, sizes ...>::type, SZ> type;
+};
+
+template <typename T, size_t ... sizes>
+using make_vector = typename matryoshka<T, sizes ...>::type;
+    typedef make_vector<int, 2, 3> iiv_t;
+};
+#endif
+
+namespace using_problem {
+
+template<typename T>
+class Base {
+public:
+    template<typename R>
+    R get1(T t) { return t + R{5}; }
+    T get2() { return T{5}; }
+    template<typename R>
+    R get3(T t) { return t + R{5}; }
+    T get3() { return T{5}; }
+};
+
+template<typename T>
+class Derived : public Base<T> {
+public:
+    typedef Base<T> _Mybase;
+    using _Mybase::get1;
+    using _Mybase::get2;
+    using _Mybase::get3;
+};
+
+} // namespace using_problem
+
+
+//===========================================================================
+// template with r-value
+namespace T_WithRValue {
+
+template<typename T>
+bool is_valid(T&& new_value) {
+    return new_value != T{};
+}
+
+} // namespace T_WithRValue
+
+
+//===========================================================================
+// variadic templates
+namespace some_variadic {
+
+#ifdef WIN32
+extern __declspec(dllimport) std::string gTypeName;
+#else
+extern std::string gTypeName;
+#endif
+
+template <typename ... Args>
+class A {
+public:
+    A() {
+        gTypeName = demangle_it(typeid(A<Args...>).name(), "A::A");
+    }
+    A(const A&) = default;
+    A(A&&) = default;
+    A& operator=(const A&) = default;
+    A& operator=(A&&) = default;
+
+    template <typename ... FArgs>
+    void a(FArgs&&... args) {
+        gTypeName = demangle_it(typeid(&A<Args...>::a<FArgs...>).name(), "A::a-2");
+    }
+
+    template <typename T, typename ... FArgs>
+    T a_T(FArgs&&... args) {
+        gTypeName = demangle_it(typeid(&A<Args...>::a_T<T, FArgs...>).name(), "A::a_T-2");
+        return T{};
+    }
+
+    template <typename ... FArgs>
+    static void sa(FArgs&&... args) {
+        gTypeName = demangle_it(typeid(A<Args...>).name(), "A::sa-1");
+        gTypeName += "::";
+        gTypeName += demangle_it(typeid(A<Args...>::sa<FArgs...>).name(), "A::sa-2");
+    }
+
+    template <typename T, typename ... FArgs>
+    static T sa_T(FArgs&&... args) {
+        gTypeName = demangle_it(typeid(A<Args...>).name(), "A::sa_T-1");
+        gTypeName +=  "::";
+        gTypeName += demangle_it(typeid(A<Args...>::sa_T<T, FArgs...>).name(), "A::sa_T-2");
+        return T{};
+    }
+};
+
+class B {
+public:
+    B() {
+        gTypeName = demangle_it(typeid(B).name(), "B::B");
+    }
+    B(const B&) = default;
+    B(B&&) = default;
+    B& operator=(const B&) = default;
+    B& operator=(B&&) = default;
+
+    template <typename ... FArgs>
+    void b(FArgs&&... args) {
+        gTypeName = demangle_it(typeid(&B::b<FArgs...>).name(), "B::b-2");
+    }
+
+    template <typename T, typename ... FArgs>
+    T b_T(FArgs&&... args) {
+        gTypeName = demangle_it(typeid(&B::b_T<T, FArgs...>).name(), "B::b_T-2");
+        return T{};
+    }
+
+    template <typename ... FArgs>
+    static void sb(FArgs&&... args) {
+        gTypeName = demangle_it(typeid(B).name(), "B::sb-1");
+        gTypeName += "::";
+        gTypeName +=  demangle_it(typeid(B::sb<FArgs...>).name(), "B::sb-2");
+    }
+
+    template <typename T, typename ... FArgs>
+    static T sb_T(FArgs&&... args) {
+        gTypeName = demangle_it(typeid(B).name(), "B::sb_T-1");
+        gTypeName += "::";
+        gTypeName += demangle_it(typeid(B::sb_T<T, FArgs...>).name(), "B::sb_T-2");
+        return T{};
+    }
+};
+
+template <typename ... Args>
+void fn(Args&&... args) {
+    gTypeName = demangle_it(typeid(fn<Args...>).name(), "fn");
+}
+
+template <typename T, typename ... Args>
+T fn_T(Args&&... args) {
+    gTypeName = demangle_it(typeid(fn<Args...>).name(), "fn_T");
+    return T{};
+}
+
+} // namespace some_variadic
+
+
+//===========================================================================
+// template with empty body
+namespace T_WithEmptyBody {
+
+#ifdef WIN32
+extern __declspec(dllimport) std::string side_effect;
+#else
+extern std::string side_effect;
+#endif
+
+template<typename T>
+void some_empty();
+
+} // namespace T_WithEmptyBody
+
+
+//===========================================================================
+// template with catch-all (void*, void**)overloads
+namespace T_WithGreedyOverloads {
+
+class SomeClass {
+    double fD;
+};
+
+class WithGreedy1 {
+public:
+    template<class T>
+    int get_size(T*) { return (int)sizeof(T); }
+    int get_size(void*, bool force=false) { return -1; }
+};
+
+class WithGreedy2 {
+public:
+    template<class T>
+    int get_size(T*) { return (int)sizeof(T); }
+    int get_size(void**, bool force=false) { return -1; }
+};
+
+class DoesNotExist;
+
+class WithGreedy3 {
+public:
+    template<class T>
+    int get_size(T*) { return (int)sizeof(T); }
+    int get_size(DoesNotExist*, bool force=false) { return -1; }
+};
+
+} // namespace T_WithGreedyOverloads
+
+
+//===========================================================================
+// template with overloaded non-templated and templated setitem
+namespace TemplateWithSetItem {
+
+template <typename T>
+class MyVec {
+private:
+   std::vector<T> fData;
+
+public:
+   using size_type = typename std::vector<T>::size_type;
+
+   MyVec(size_type count) : fData(count) {}
+
+   T & operator[](size_type index) { return fData[index]; }
+
+   // The definition of this templated operator causes the issue
+   template <typename V>
+   MyVec operator[](const MyVec<V> &conds) const { return MyVec(2); }
+};
+
+} // namespace TemplateWithSetItem
+
+#endif // !CPPYY_TEST_TEMPLATES_H
diff --git a/pypy/module/_cppyy/test/templates.xml b/pypy/module/_cppyy/test/templates.xml
--- a/pypy/module/_cppyy/test/templates.xml
+++ b/pypy/module/_cppyy/test/templates.xml
@@ -16,4 +16,21 @@
   <function name="SomeNS::some_bar" />
   <function name="SomeNS::tuplify" />
 
+  <class name="TemplatedCallable" />
+
+  <namespace name="TemplateHiding" />
+  <class pattern="TemplateHiding::*" />
+
+  <namespace name="using_problem" />
+
+  <namespace name="some_variadic" />
+  <class pattern="some_variadic::*" />
+  <function pattern="some_variadic::*" />
+  <variable pattern="some_variadic::*" />
+
+  <namespace name="T_WithEmptyBody" />
+
+  <namespace name="T_WithGreedyOverloads" />
+  <class pattern="T_WithGreedyOverloads::*" />
+
 </lcgdict>
diff --git a/pypy/module/_cppyy/test/test_fragile.py b/pypy/module/_cppyy/test/test_fragile.py
--- a/pypy/module/_cppyy/test/test_fragile.py
+++ b/pypy/module/_cppyy/test/test_fragile.py
@@ -35,7 +35,7 @@
 
         assert fragile.B == fragile.B
         assert fragile.B().check() == ord('B')
-        raises(AttributeError, getattr, fragile.B().gime_no_such(), "__cppdecl__")
+        assert not fragile.B().gime_no_such()
 
         assert fragile.C == fragile.C
         assert fragile.C().check() == ord('C')
diff --git a/pypy/module/_cppyy/test/test_regression.py b/pypy/module/_cppyy/test/test_regression.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_cppyy/test/test_regression.py
@@ -0,0 +1,50 @@
+import py, os, sys
+from .support import setup_make
+
+from pypy.module._cppyy import interp_cppyy, executor
+
+
+class AppTestREGRESSION:
+    spaceconfig = dict(usemodules=['_cppyy', '_rawffi', 'itertools'])
+
+    def setup_class(cls):
+        cls.w_example01 = cls.space.appexec([], """():
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()""")
+
+    def test01_dir(self):
+        """These dir() methods used to crash."""
+
+        import _cppyy as cppyy
+
+        cppyy.gbl.gInterpreter.Declare("namespace cppyy_regression_test { void iii() {}; }")
+
+        assert not 'iii' in cppyy.gbl.cppyy_regression_test.__dict__
+        assert not '__abstractmethods__' in dir(cppyy.gbl.cppyy_regression_test)
+        assert '__class__' in dir(cppyy.gbl.cppyy_regression_test)
+        assert 'iii' in dir(cppyy.gbl.cppyy_regression_test)
+
+        assert not 'iii' in cppyy.gbl.cppyy_regression_test.__dict__
+        assert cppyy.gbl.cppyy_regression_test.iii
+        assert 'iii' in cppyy.gbl.cppyy_regression_test.__dict__
+
+    def test02_default_template_arguments(self):
+        """Calling a templated method on a templated class with all defaults used to crash."""
+
+        import _cppyy as cppyy
+
+        cppyy.gbl.gInterpreter.Declare("""
+        template<typename T>
+        class AllDefault {
+        public:
+           AllDefault(int val) : m_t(val) {}
+           template<int aap=1, int noot=2>
+              int do_stuff() { return m_t+aap+noot; }
+
+        public:
+           T m_t;
+        };""")
+
+        a = cppyy.gbl.AllDefault[int](24)
+        a.m_t = 21;
+        assert a.do_stuff() == 24
diff --git a/pypy/module/_cppyy/test/test_templates.py b/pypy/module/_cppyy/test/test_templates.py
--- a/pypy/module/_cppyy/test/test_templates.py
+++ b/pypy/module/_cppyy/test/test_templates.py
@@ -25,20 +25,35 @@
 
         m = _cppyy.gbl.MyTemplatedMethodClass()
 
+      # implicit (called before other tests to check caching)
+        assert m.get_size(1)          == m.get_int_size()+1
+        assert 'get_size<int>' in dir(_cppyy.gbl.MyTemplatedMethodClass)
+
       # pre-instantiated
         assert m.get_size['char']()   == m.get_char_size()
         assert m.get_size[int]()      == m.get_int_size()
 
       # specialized
-        assert m.get_size[long]()     == m.get_long_size()
+        if sys.hexversion >= 0x3000000:
+            targ = 'long'
+        else:
+            targ = long
+        assert m.get_size[targ]()     == m.get_long_size()
+
+        import ctypes
+        assert m.get_size(ctypes.c_double(3.14)) == m.get_size['double']()
+        assert m.get_size(ctypes.c_double(3.14).value) == m.get_size['double']()+1
 
       # auto-instantiation
         assert m.get_size[float]()    == m.get_float_size()
         assert m.get_size['double']() == m.get_double_size()
         assert m.get_size['MyTemplatedMethodClass']() == m.get_self_size()
+        assert 'get_size<MyTemplatedMethodClass>' in dir(_cppyy.gbl.MyTemplatedMethodClass)
 
       # auto through typedef
         assert m.get_size['MyTMCTypedef_t']() == m.get_self_size()
+        assert 'get_size<MyTMCTypedef_t>' in dir(_cppyy.gbl.MyTemplatedMethodClass)
+        assert m.get_size['MyTemplatedMethodClass']() == m.get_self_size()
 
     def test02_non_type_template_args(self):
         """Use of non-types as template arguments"""
@@ -90,15 +105,13 @@
         assert type(ggsr(vector['int']([5])).m_retval) == float
         assert ggsr(vector['int']([5])).m_retval == 5.
         # float in, int out
-        # TODO: this now matches the earlier overload
-        #ggsr = cppyy.gbl.global_get_some_result['std::vector<float>, int']
-        #assert type(ggsr(vector['float']([0.3])).m_retval) == int
-        #assert ggsr(vector['float']([0.3])).m_retval == 0
+        ggsr = cppyy.gbl.global_get_some_result['std::vector<float>, int']
+        assert type(ggsr(vector['float']([0.3])).m_retval) == int
+        assert ggsr(vector['float']([0.3])).m_retval == 0
         # int in, int out
-        # TODO: same as above, matches earlier overload
-        #ggsr = cppyy.gbl.global_get_some_result['std::vector<int>, int']
-        #assert type(ggsr(vector['int']([5])).m_retval) == int
-        #assert ggsr(vector['int']([5])).m_retval == 5
+        ggsr = cppyy.gbl.global_get_some_result['std::vector<int>, int']
+        assert type(ggsr(vector['int']([5])).m_retval) == int
+        assert ggsr(vector['int']([5])).m_retval == 5
 
     def test04_variadic_function(self):
         """Call a variadic function"""
diff --git a/pypy/module/_cppyy/test/test_zjit.py b/pypy/module/_cppyy/test/test_zjit.py
--- a/pypy/module/_cppyy/test/test_zjit.py
+++ b/pypy/module/_cppyy/test/test_zjit.py
@@ -228,6 +228,9 @@
         assert isinstance(w_obj, FakeString)
         return w_obj.val
 
+    def fsencode_w(self, w_obj):
+        return self.bytes_w(w_obj)
+
     def text_w(self, w_obj):
         assert isinstance(w_obj, FakeString)
         return w_obj.val
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -32,8 +32,8 @@
  *     module/sys/version.py
  *     doc/conf.py
  */
-#define PYPY_VERSION "7.3.0-alpha0"
-#define PYPY_VERSION_NUM  0x07030000
+#define PYPY_VERSION "7.3.1-alpha0"
+#define PYPY_VERSION_NUM  0x07030100
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
    stays alive. */
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -13,7 +13,7 @@
 # make sure to keep PYPY_VERSION in sync with:
 #    module/cpyext/include/patchlevel.h
 #    doc/conf.py
-PYPY_VERSION               = (7, 3, 0, "alpha", 0)
+PYPY_VERSION               = (7, 3, 1, "alpha", 0)
 
 
 import pypy

From pypy.commits at gmail.com  Wed Nov 27 02:01:53 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 26 Nov 2019 23:01:53 -0800 (PST)
Subject: [pypy-commit] pypy release-pypy3.6-v7.x: merge py3.6 into
 release-pypy3
Message-ID: <5dde1f61.1c69fb81.bb09d.b4d8@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-pypy3.6-v7.x
Changeset: r98160:1249df1fb9cf
Date: 2019-11-26 08:51 -0800
http://bitbucket.org/pypy/pypy/changeset/1249df1fb9cf/

Log:	merge py3.6 into release-pypy3

diff too long, truncating to 2000 out of 13495 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -50,4 +50,10 @@
 de061d87e39c7df4e436974096d7982c676a859d release-pypy3.6-v7.1.0
 784b254d669919c872a505b807db8462b6140973 release-pypy3.6-v7.1.1
 8cdda8b8cdb8ff29d9e620cccd6c5edd2f2a23ec release-pypy2.7-v7.1.1
-
+85dae4fd5c234b482feff834c73e089872194541 release-pypy2.7-v7.2.0rc0
+7ffb92269488f37c707ce66076f50ffd8613f8e2 release-pypy3.6-v7.2.0rc0
+4d6761df14ffd6f38450f183ac1fad32c946c21b release-pypy3.6-v7.2.0rc1
+5da45ced70e515f94686be0df47c59abd1348ebc release-pypy3.6-v7.2.0rc2
+4a68d8d3d2fc1faec2e83bcb4d28559099092574 release-pypy2.7-v7.2.0rc2
+4a68d8d3d2fc1faec2e83bcb4d28559099092574 release-pypy2.7-v7.2.0
+5da45ced70e515f94686be0df47c59abd1348ebc release-pypy3.6-v7.2.0
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -3,10 +3,11 @@
 License
 =======
 
-Except when otherwise stated (look for LICENSE files in directories or
-information at the beginning of each file) all software and documentation in
-the 'rpython', 'pypy', 'ctype_configure', 'dotviewer', 'demo', 'lib_pypy',
-'py', and '_pytest' directories is licensed as follows:
+Except when otherwise stated (look for LICENSE files in directories
+or information at the beginning of each file) all software and
+documentation in the 'rpython', 'pypy', 'ctype_configure', 'dotviewer',
+'demo', 'extra_tests', 'include', 'lib_pypy', 'py', and '_pytest'
+directories is licensed as follows:
 
     The MIT License
 
@@ -98,16 +99,16 @@
   Spenser Bauman
   Michal Bendowski
   Jan de Mooij
+  Stefano Rivera
   Tyler Wade
+  Stefan Beyer
   Vincent Legoll
   Michael Foord
   Stephan Diehl
-  Stefano Rivera
   Jean-Paul Calderone
   Stefan Schwarzer
   Tomek Meka
   Valentino Volonghi
-  Stefan Beyer
   Patrick Maupin
   Devin Jeanpierre
   Bob Ippolito
@@ -136,9 +137,10 @@
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
+  Stian Andreassen
+  Julian Berman
   William Leslie
   Pawe&#322; Piotr Przeradowski
-  Stian Andreassen
   marky1991
   Ilya Osadchiy
   Tobias Oberstein
@@ -149,7 +151,7 @@
   tav
   Georg Brandl
   Joannah Nanjekye
-  Julian Berman
+  Yannick Jadoul
   Bert Freudenberg
   Wanja Saatkamp
   Mike Blume
@@ -274,6 +276,7 @@
   Lutz Paelike
   Ian Foote
   Philipp Rustemeuer
+  Bernd Schoeller
   Logan Chien
   Catalin Gabriel Manciu
   Jacob Oscarson
@@ -301,7 +304,6 @@
   Laurens Van Houtven
   Bobby Impollonia
   Roberto De Ioris
-  Yannick Jadoul
   Jeong YunWon
   Christopher Armstrong
   Aaron Tubbs
@@ -356,6 +358,7 @@
   Daniil Yarancev
   Min RK
   OlivierBlanvillain
+  bernd.schoeller at inf.ethz.ch
   dakarpov at gmail.com
   Jonas Pfannschmidt
   Zearin
@@ -397,6 +400,7 @@
   Jesdi
   Konrad Delong
   Dinu Gherman
+  Sam Edwards
   pizi
   Tom&#225;&#353; Pru&#382;ina
   James Robert
diff --git a/README.rst b/README.rst
--- a/README.rst
+++ b/README.rst
@@ -9,15 +9,15 @@
 
 The home page for the interpreter is:
 
-    http://pypy.org/
+    https://pypy.org/
 
 If you want to help developing PyPy, this documentation might help you:
 
-    http://doc.pypy.org/
+    https://doc.pypy.org/
 
 More documentation about the RPython framework can be found here:
 
-    http://rpython.readthedocs.io/
+    https://rpython.readthedocs.io/
 
 The source for the documentation is in the pypy/doc directory.
 
@@ -25,7 +25,7 @@
 Using PyPy instead of CPython
 -----------------------------
 
-Please read the information at http://pypy.org/ to find the correct way to
+Please read the information at https://pypy.org/ to find the correct way to
 download and use PyPy as an alternative to CPython. 
 
 
@@ -36,7 +36,7 @@
 interpreter. It is time-consuming and requires significant computing resources.
 More information can be found here:
 
-    http://doc.pypy.org/en/latest/build.html
+    https://doc.pypy.org/en/latest/build.html
 
 Enjoy and send us feedback!
 
diff --git a/extra_tests/cffi_tests/cffi0/test_function.py b/extra_tests/cffi_tests/cffi0/test_function.py
--- a/extra_tests/cffi_tests/cffi0/test_function.py
+++ b/extra_tests/cffi_tests/cffi0/test_function.py
@@ -114,7 +114,7 @@
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
             int fputs(const char *, void *);
-            void *stderr;
+            extern void *stderr;
         """)
         needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
@@ -131,7 +131,7 @@
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
             int fputs(char *, void *);
-            void *stderr;
+            extern void *stderr;
         """)
         needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
@@ -148,7 +148,7 @@
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
            int fprintf(void *, const char *format, ...);
-           void *stderr;
+           extern void *stderr;
         """)
         needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
@@ -210,7 +210,7 @@
             py.test.skip("probably no symbol 'stderr' in the lib")
         ffi.cdef("""
             int fputs(const char *, void *);
-            void *stderr;
+            extern void *stderr;
         """)
         needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
@@ -257,7 +257,7 @@
             py.test.skip("probably no symbol 'stdout' in the lib")
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
-            void *stdout;
+            extern void *stdout;
         """)
         needs_dlopen_none()
         C = ffi.dlopen(None)
@@ -497,7 +497,7 @@
         ffi.cdef("""
             typedef enum { MYE1, MYE2 } myenum_t;
             double myfunc(double);
-            double myvar;
+            extern double myvar;
             const double myconst;
             #define MYFOO 42
         """)
@@ -508,7 +508,7 @@
         if self.Backend is CTypesBackend:
             py.test.skip("not with the ctypes backend")
         ffi = FFI(backend=self.Backend())
-        ffi.cdef("int foobar(void); int foobaz;")
+        ffi.cdef("int foobar(void); extern int foobaz;")
         lib = ffi.dlopen(lib_m)
         ffi.dlclose(lib)
         e = py.test.raises(ValueError, getattr, lib, 'foobar')
diff --git a/extra_tests/cffi_tests/cffi0/test_ownlib.py b/extra_tests/cffi_tests/cffi0/test_ownlib.py
--- a/extra_tests/cffi_tests/cffi0/test_ownlib.py
+++ b/extra_tests/cffi_tests/cffi0/test_ownlib.py
@@ -202,7 +202,7 @@
             py.test.skip("fix the auto-generation of the tiny test lib")
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
-            int my_array[7];
+            extern int my_array[7];
         """)
         ownlib = ffi.dlopen(self.module)
         for i in range(7):
@@ -224,7 +224,7 @@
             py.test.skip("not supported by the ctypes backend")
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
-            int my_array[];
+            extern int my_array[];
         """)
         ownlib = ffi.dlopen(self.module)
         for i in range(7):
@@ -292,7 +292,7 @@
                 long bottom;
             } RECT;
             
-            long left, top, right, bottom;
+            extern long left, top, right, bottom;
 
             RECT ReturnRect(int i, RECT ar, RECT* br, POINT cp, RECT dr,
                         RECT *er, POINT fp, RECT gr);
@@ -322,7 +322,7 @@
         if self.Backend is CTypesBackend:
             py.test.skip("not implemented with the ctypes backend")
         ffi = FFI(backend=self.Backend())
-        ffi.cdef("long left; int test_getting_errno(void);")
+        ffi.cdef("extern long left; int test_getting_errno(void);")
         lib = ffi.dlopen(self.module)
         lib.left = 123456
         p = ffi.addressof(lib, "left")
diff --git a/extra_tests/cffi_tests/cffi0/test_parsing.py b/extra_tests/cffi_tests/cffi0/test_parsing.py
--- a/extra_tests/cffi_tests/cffi0/test_parsing.py
+++ b/extra_tests/cffi_tests/cffi0/test_parsing.py
@@ -325,6 +325,7 @@
     assert value == sys.maxsize * 2 - 40
 
 def test__is_constant_globalvar():
+    import warnings
     for input, expected_output in [
         ("int a;",          False),
         ("const int a;",    True),
@@ -342,10 +343,13 @@
         ("const int a[5][6];", False),
         ]:
         ffi = FFI()
-        ffi.cdef(input)
+        with warnings.catch_warnings(record=True) as log:
+            warnings.simplefilter("always")
+            ffi.cdef(input)
         declarations = ffi._parser._declarations
         assert ('constant a' in declarations) == expected_output
         assert ('variable a' in declarations) == (not expected_output)
+        assert len(log) == (1 - expected_output)
 
 def test_restrict():
     from cffi import model
@@ -355,7 +359,7 @@
         ("int *a;",            False),
         ]:
         ffi = FFI()
-        ffi.cdef(input)
+        ffi.cdef("extern " + input)
         tp, quals = ffi._parser._declarations['variable a']
         assert bool(quals & model.Q_RESTRICT) == expected_output
 
diff --git a/extra_tests/cffi_tests/cffi0/test_verify.py b/extra_tests/cffi_tests/cffi0/test_verify.py
--- a/extra_tests/cffi_tests/cffi0/test_verify.py
+++ b/extra_tests/cffi_tests/cffi0/test_verify.py
@@ -4,6 +4,7 @@
 import sys, os, math, weakref
 from cffi import FFI, VerificationError, VerificationMissing, model, FFIError
 from extra_tests.cffi_tests.support import *
+from extra_tests.cffi_tests.support import extra_compile_args
 
 
 lib_m = ['m']
@@ -14,17 +15,6 @@
         lib_m = ['msvcrt']
     pass      # no obvious -Werror equivalent on MSVC
 else:
-    if (sys.platform == 'darwin' and
-          [int(x) for x in os.uname()[2].split('.')] >= [11, 0, 0]):
-        # assume a standard clang or gcc
-        extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion']
-        # special things for clang
-        extra_compile_args.append('-Qunused-arguments')
-    else:
-        # assume a standard gcc
-        extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion',
-                              '-Wno-unused-parameter']
-
     class FFI(FFI):
         def verify(self, *args, **kwds):
             return super(FFI, self).verify(
@@ -287,7 +277,7 @@
 def test_var_signed_integer_types():
     ffi = FFI()
     lst = all_signed_integer_types(ffi)
-    csource = "\n".join(["%s somevar_%s;" % (tp, tp.replace(' ', '_'))
+    csource = "\n".join(["static %s somevar_%s;" % (tp, tp.replace(' ', '_'))
                          for tp in lst])
     ffi.cdef(csource)
     lib = ffi.verify(csource)
@@ -306,7 +296,7 @@
 def test_var_unsigned_integer_types():
     ffi = FFI()
     lst = all_unsigned_integer_types(ffi)
-    csource = "\n".join(["%s somevar_%s;" % (tp, tp.replace(' ', '_'))
+    csource = "\n".join(["static %s somevar_%s;" % (tp, tp.replace(' ', '_'))
                          for tp in lst])
     ffi.cdef(csource)
     lib = ffi.verify(csource)
@@ -818,8 +808,8 @@
 
 def test_access_variable():
     ffi = FFI()
-    ffi.cdef("int foo(void);\n"
-             "int somenumber;")
+    ffi.cdef("static int foo(void);\n"
+             "static int somenumber;")
     lib = ffi.verify("""
         static int somenumber = 2;
         static int foo(void) {
@@ -836,7 +826,7 @@
 def test_access_address_of_variable():
     # access the address of 'somenumber': need a trick
     ffi = FFI()
-    ffi.cdef("int somenumber; static int *const somenumberptr;")
+    ffi.cdef("static int somenumber; static int *const somenumberptr;")
     lib = ffi.verify("""
         static int somenumber = 2;
         #define somenumberptr (&somenumber)
@@ -849,7 +839,7 @@
 def test_access_array_variable(length=5):
     ffi = FFI()
     ffi.cdef("int foo(int);\n"
-             "int somenumber[%s];" % (length,))
+             "static int somenumber[%s];" % (length,))
     lib = ffi.verify("""
         static int somenumber[] = {2, 2, 3, 4, 5};
         static int foo(int i) {
@@ -881,7 +871,7 @@
     ffi = FFI()
     ffi.cdef("struct foo { int x; ...; };\n"
              "int foo(int);\n"
-             "struct foo stuff;")
+             "static struct foo stuff;")
     lib = ffi.verify("""
         struct foo { int x, y, z; };
         static struct foo stuff = {2, 5, 8};
@@ -905,9 +895,9 @@
 
 def test_access_callback():
     ffi = FFI()
-    ffi.cdef("int (*cb)(int);\n"
-             "int foo(int);\n"
-             "void reset_cb(void);")
+    ffi.cdef("static int (*cb)(int);\n"
+             "static int foo(int);\n"
+             "static void reset_cb(void);")
     lib = ffi.verify("""
         static int g(int x) { return x * 7; }
         static int (*cb)(int);
@@ -923,9 +913,9 @@
 def test_access_callback_function_typedef():
     ffi = FFI()
     ffi.cdef("typedef int mycallback_t(int);\n"
-             "mycallback_t *cb;\n"
-             "int foo(int);\n"
-             "void reset_cb(void);")
+             "static mycallback_t *cb;\n"
+             "static int foo(int);\n"
+             "static void reset_cb(void);")
     lib = ffi.verify("""
         static int g(int x) { return x * 7; }
         static int (*cb)(int);
@@ -1075,7 +1065,7 @@
 def test_autofilled_struct_as_argument_dynamic():
     ffi = FFI()
     ffi.cdef("struct foo_s { long a; ...; };\n"
-             "int (*foo)(struct foo_s);")
+             "static int (*foo)(struct foo_s);")
     lib = ffi.verify("""
         struct foo_s {
             double b;
@@ -1084,7 +1074,7 @@
         int foo1(struct foo_s s) {
             return (int)s.a - (int)s.b;
         }
-        int (*foo)(struct foo_s s) = &foo1;
+        static int (*foo)(struct foo_s s) = &foo1;
     """)
     e = py.test.raises(NotImplementedError, lib.foo, "?")
     msg = ("ctype 'struct foo_s' not supported as argument.  It is a struct "
@@ -1454,7 +1444,7 @@
         py.test.skip("_Bool not in MSVC")
     ffi = FFI()
     ffi.cdef("struct foo_s { _Bool x; };"
-             "_Bool foo(_Bool); _Bool (*foop)(_Bool);")
+             "_Bool foo(_Bool); static _Bool (*foop)(_Bool);")
     lib = ffi.verify("""
         struct foo_s { _Bool x; };
         int foo(int arg) {
@@ -1463,7 +1453,7 @@
         _Bool _foofunc(_Bool x) {
             return !x;
         }
-        _Bool (*foop)(_Bool) = _foofunc;
+        static _Bool (*foop)(_Bool) = _foofunc;
     """)
     p = ffi.new("struct foo_s *")
     p.x = 1
@@ -1654,7 +1644,7 @@
 
 def test_FILE_stored_explicitly():
     ffi = FFI()
-    ffi.cdef("int myprintf11(const char *, int); FILE *myfile;")
+    ffi.cdef("int myprintf11(const char *, int); extern FILE *myfile;")
     lib = ffi.verify("""
         #include <stdio.h>
         FILE *myfile;
@@ -1680,19 +1670,19 @@
 
 def test_global_array_with_missing_length():
     ffi = FFI()
-    ffi.cdef("int fooarray[];")
+    ffi.cdef("extern int fooarray[];")
     lib = ffi.verify("int fooarray[50];")
     assert repr(lib.fooarray).startswith("<cdata 'int *'")
 
 def test_global_array_with_dotdotdot_length():
     ffi = FFI()
-    ffi.cdef("int fooarray[...];")
+    ffi.cdef("extern int fooarray[...];")
     lib = ffi.verify("int fooarray[50];")
     assert repr(lib.fooarray).startswith("<cdata 'int[50]'")
 
 def test_bad_global_array_with_dotdotdot_length():
     ffi = FFI()
-    ffi.cdef("int fooarray[...];")
+    ffi.cdef("extern int fooarray[...];")
     py.test.raises(VerificationError, ffi.verify, "char fooarray[23];")
 
 def test_struct_containing_struct():
@@ -1813,7 +1803,7 @@
 def test_callback_indirection():
     ffi = FFI()
     ffi.cdef("""
-        int (*python_callback)(int how_many, int *values);
+        static int (*python_callback)(int how_many, int *values);
         int (*const c_callback)(int,...);   /* pass this ptr to C routines */
         int some_c_function(int(*cb)(int,...));
     """)
@@ -1947,24 +1937,24 @@
 
 def test_bug_const_char_ptr_array_1():
     ffi = FFI()
-    ffi.cdef("""const char *a[...];""")
+    ffi.cdef("""extern const char *a[...];""")
     lib = ffi.verify("""const char *a[5];""")
     assert repr(ffi.typeof(lib.a)) == "<ctype 'char *[5]'>"
 
 def test_bug_const_char_ptr_array_2():
     from cffi import FFI     # ignore warnings
     ffi = FFI()
-    ffi.cdef("""const int a[];""")
+    ffi.cdef("""extern const int a[];""")
     lib = ffi.verify("""const int a[5];""")
     assert repr(ffi.typeof(lib.a)) == "<ctype 'int *'>"
 
 def _test_various_calls(force_libffi):
     cdef_source = """
-    int xvalue;
-    long long ivalue, rvalue;
-    float fvalue;
-    double dvalue;
-    long double Dvalue;
+    extern int xvalue;
+    extern long long ivalue, rvalue;
+    extern float fvalue;
+    extern double dvalue;
+    extern long double Dvalue;
     signed char tf_bb(signed char x, signed char c);
     unsigned char tf_bB(signed char x, unsigned char c);
     short tf_bh(signed char x, short c);
@@ -2148,7 +2138,7 @@
     # exported symbols as well.  So we must not export a simple name
     # like 'foo'!
     ffi1 = FFI()
-    ffi1.cdef("int foo_verify_dlopen_flags;")
+    ffi1.cdef("extern int foo_verify_dlopen_flags;")
 
     lib1 = ffi1.verify("int foo_verify_dlopen_flags;",
                        flags=ffi1.RTLD_GLOBAL | ffi1.RTLD_LAZY)
@@ -2162,7 +2152,7 @@
 def get_second_lib():
     # Hack, using modulename makes the test fail
     ffi2 = FFI()
-    ffi2.cdef("int foo_verify_dlopen_flags;")
+    ffi2.cdef("extern int foo_verify_dlopen_flags;")
     lib2 = ffi2.verify("int foo_verify_dlopen_flags;",
                        flags=ffi2.RTLD_GLOBAL | ffi2.RTLD_LAZY)
     return lib2
diff --git a/extra_tests/cffi_tests/cffi1/test_dlopen.py b/extra_tests/cffi_tests/cffi1/test_dlopen.py
--- a/extra_tests/cffi_tests/cffi1/test_dlopen.py
+++ b/extra_tests/cffi_tests/cffi1/test_dlopen.py
@@ -7,7 +7,7 @@
 
 def test_simple():
     ffi = FFI()
-    ffi.cdef("int close(int); static const int BB = 42; int somevar;")
+    ffi.cdef("int close(int); static const int BB = 42; extern int somevar;")
     target = udir.join('test_simple.py')
     make_py_source(ffi, 'test_simple', str(target))
     assert target.read() == r"""# auto-generated file
@@ -197,7 +197,7 @@
 
 def test_global_var():
     ffi = FFI()
-    ffi.cdef("int myglob;")
+    ffi.cdef("extern int myglob;")
     target = udir.join('test_global_var.py')
     make_py_source(ffi, 'test_global_var', str(target))
     assert target.read() == r"""# auto-generated file
diff --git a/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py
--- a/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py
+++ b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py
@@ -1780,7 +1780,7 @@
 
     def test_import_from_lib(self):
         ffi2 = cffi.FFI()
-        ffi2.cdef("int myfunc(int); int myvar;\n#define MYFOO ...\n")
+        ffi2.cdef("int myfunc(int); extern int myvar;\n#define MYFOO ...\n")
         outputfilename = recompile(ffi2, "_test_import_from_lib",
                                    "int myfunc(int x) { return x + 1; }\n"
                                    "int myvar = -5;\n"
diff --git a/extra_tests/cffi_tests/cffi1/test_re_python.py b/extra_tests/cffi_tests/cffi1/test_re_python.py
--- a/extra_tests/cffi_tests/cffi1/test_re_python.py
+++ b/extra_tests/cffi_tests/cffi1/test_re_python.py
@@ -64,11 +64,11 @@
     #define BIGNEG -420000000000L
     int add42(int);
     int add43(int, ...);
-    int globalvar42;
+    extern int globalvar42;
     const int globalconst42;
     const char *const globalconsthello;
     int no_such_function(int);
-    int no_such_globalvar;
+    extern int no_such_globalvar;
     struct foo_s;
     typedef struct bar_s { int x; signed char a[]; } bar_t;
     enum foo_e { AA, BB, CC };
@@ -76,6 +76,7 @@
     struct with_union { union { int a; char b; }; };
     union with_struct { struct { int a; char b; }; };
     struct NVGcolor { union { float rgba[4]; struct { float r,g,b,a; }; }; };
+    typedef struct selfref { struct selfref *next; } *selfref_ptr_t;
     """)
     ffi.set_source('re_python_pysrc', None)
     ffi.emit_python_code(str(tmpdir.join('re_python_pysrc.py')))
@@ -255,3 +256,8 @@
     assert ffi.offsetof("struct NVGcolor", "g") == FLOAT
     assert ffi.offsetof("struct NVGcolor", "b") == FLOAT * 2
     assert ffi.offsetof("struct NVGcolor", "a") == FLOAT * 3
+
+def test_selfref():
+    # based on issue #429
+    from re_python_pysrc import ffi
+    ffi.new("selfref_ptr_t")
diff --git a/extra_tests/cffi_tests/cffi1/test_recompiler.py b/extra_tests/cffi_tests/cffi1/test_recompiler.py
--- a/extra_tests/cffi_tests/cffi1/test_recompiler.py
+++ b/extra_tests/cffi_tests/cffi1/test_recompiler.py
@@ -35,8 +35,9 @@
         source = 'extern "C" {\n%s\n}' % (source,)
     elif sys.platform != 'win32':
         # add '-Werror' to the existing 'extra_compile_args' flags
+        from extra_tests.cffi_tests.support import extra_compile_args
         kwds['extra_compile_args'] = (kwds.get('extra_compile_args', []) +
-                                      ['-Werror'])
+                                      extra_compile_args)
     return _verify(ffi, module_name, source, *args, **kwds)
 
 def test_set_source_no_slashes():
@@ -84,7 +85,7 @@
                      "(FUNCTION 1)(PRIMITIVE 7)(FUNCTION_END 1)(POINTER 0)")
 
 def test_type_table_array():
-    check_type_table("int a[100];",
+    check_type_table("extern int a[100];",
                      "(PRIMITIVE 7)(ARRAY 0)(None 100)")
 
 def test_type_table_typedef():
@@ -159,7 +160,7 @@
 
 def test_global_var_array():
     ffi = FFI()
-    ffi.cdef("int a[100];")
+    ffi.cdef("extern int a[100];")
     lib = verify(ffi, 'test_global_var_array', 'int a[100] = { 9999 };')
     lib.a[42] = 123456
     assert lib.a[42] == 123456
@@ -183,7 +184,7 @@
 
 def test_global_var_int():
     ffi = FFI()
-    ffi.cdef("int a, b, c;")
+    ffi.cdef("extern int a, b, c;")
     lib = verify(ffi, 'test_global_var_int', 'int a = 999, b, c;')
     assert lib.a == 999
     lib.a -= 1001
@@ -284,7 +285,7 @@
 
 def test_dir():
     ffi = FFI()
-    ffi.cdef("int ff(int); int aa; static const int my_constant;")
+    ffi.cdef("int ff(int); extern int aa; static const int my_constant;")
     lib = verify(ffi, 'test_dir', """
         #define my_constant  (-45)
         int aa;
@@ -406,7 +407,7 @@
 
 def test_dotdotdot_global_array():
     ffi = FFI()
-    ffi.cdef("int aa[...]; int bb[...];")
+    ffi.cdef("extern int aa[...]; extern int bb[...];")
     lib = verify(ffi, 'test_dotdotdot_global_array',
                  "int aa[41]; int bb[12];")
     assert ffi.sizeof(lib.aa) == 41 * 4
@@ -561,37 +562,37 @@
 
 def test_bad_size_of_global_1():
     ffi = FFI()
-    ffi.cdef("short glob;")
+    ffi.cdef("extern short glob;")
     py.test.raises(VerificationError, verify, ffi,
                    "test_bad_size_of_global_1", "long glob;")
 
 def test_bad_size_of_global_2():
     ffi = FFI()
-    ffi.cdef("int glob[10];")
+    ffi.cdef("extern int glob[10];")
     py.test.raises(VerificationError, verify, ffi,
                    "test_bad_size_of_global_2", "int glob[9];")
 
 def test_unspecified_size_of_global_1():
     ffi = FFI()
-    ffi.cdef("int glob[];")
+    ffi.cdef("extern int glob[];")
     lib = verify(ffi, "test_unspecified_size_of_global_1", "int glob[10];")
     assert ffi.typeof(lib.glob) == ffi.typeof("int *")
 
 def test_unspecified_size_of_global_2():
     ffi = FFI()
-    ffi.cdef("int glob[][5];")
+    ffi.cdef("extern int glob[][5];")
     lib = verify(ffi, "test_unspecified_size_of_global_2", "int glob[10][5];")
     assert ffi.typeof(lib.glob) == ffi.typeof("int(*)[5]")
 
 def test_unspecified_size_of_global_3():
     ffi = FFI()
-    ffi.cdef("int glob[][...];")
+    ffi.cdef("extern int glob[][...];")
     lib = verify(ffi, "test_unspecified_size_of_global_3", "int glob[10][5];")
     assert ffi.typeof(lib.glob) == ffi.typeof("int(*)[5]")
 
 def test_unspecified_size_of_global_4():
     ffi = FFI()
-    ffi.cdef("int glob[...][...];")
+    ffi.cdef("extern int glob[...][...];")
     lib = verify(ffi, "test_unspecified_size_of_global_4", "int glob[10][5];")
     assert ffi.typeof(lib.glob) == ffi.typeof("int[10][5]")
 
@@ -814,7 +815,7 @@
 def test_address_of_global_var():
     ffi = FFI()
     ffi.cdef("""
-        long bottom, bottoms[2];
+        extern long bottom, bottoms[2];
         long FetchRectBottom(void);
         long FetchRectBottoms1(void);
         #define FOOBAR 42
@@ -969,7 +970,7 @@
     ffi = FFI()
     ffi.cdef("""
         typedef ... opaque_t;
-        opaque_t globvar;
+        extern opaque_t globvar;
     """)
     lib = verify(ffi, 'test_variable_of_unknown_size', """
         typedef char opaque_t[6];
@@ -1014,7 +1015,7 @@
 def test_call_with_incomplete_structs():
     ffi = FFI()
     ffi.cdef("typedef struct {...;} foo_t; "
-             "foo_t myglob; "
+             "extern foo_t myglob; "
              "foo_t increment(foo_t s); "
              "double getx(foo_t s);")
     lib = verify(ffi, 'test_call_with_incomplete_structs', """
@@ -1058,7 +1059,7 @@
 
 def test_global_var_array_2():
     ffi = FFI()
-    ffi.cdef("int a[...][...];")
+    ffi.cdef("extern int a[...][...];")
     lib = verify(ffi, 'test_global_var_array_2', 'int a[10][8];')
     lib.a[9][7] = 123456
     assert lib.a[9][7] == 123456
@@ -1071,7 +1072,7 @@
 
 def test_global_var_array_3():
     ffi = FFI()
-    ffi.cdef("int a[][...];")
+    ffi.cdef("extern int a[][...];")
     lib = verify(ffi, 'test_global_var_array_3', 'int a[10][8];')
     lib.a[9][7] = 123456
     assert lib.a[9][7] == 123456
@@ -1082,7 +1083,7 @@
 
 def test_global_var_array_4():
     ffi = FFI()
-    ffi.cdef("int a[10][...];")
+    ffi.cdef("extern int a[10][...];")
     lib = verify(ffi, 'test_global_var_array_4', 'int a[10][8];')
     lib.a[9][7] = 123456
     assert lib.a[9][7] == 123456
@@ -1205,7 +1206,7 @@
 
 def test_import_from_lib():
     ffi = FFI()
-    ffi.cdef("int mybar(int); int myvar;\n#define MYFOO ...")
+    ffi.cdef("int mybar(int); static int myvar;\n#define MYFOO ...")
     lib = verify(ffi, 'test_import_from_lib',
                  "#define MYFOO 42\n"
                  "static int mybar(int x) { return x + 1; }\n"
@@ -1221,7 +1222,7 @@
 
 def test_macro_var_callback():
     ffi = FFI()
-    ffi.cdef("int my_value; int *(*get_my_value)(void);")
+    ffi.cdef("extern int my_value; extern int *(*get_my_value)(void);")
     lib = verify(ffi, 'test_macro_var_callback',
                  "int *(*get_my_value)(void);\n"
                  "#define my_value (*get_my_value())")
@@ -1336,7 +1337,7 @@
 
 def test_const_function_type_args():
     ffi = FFI()
-    ffi.cdef("""int (*foobar)(const int a, const int *b, const int c[]);""")
+    ffi.cdef("""extern int(*foobar)(const int a,const int*b,const int c[]);""")
     lib = verify(ffi, 'test_const_function_type_args', """
         int (*foobar)(const int a, const int *b, const int c[]);
     """)
@@ -1626,7 +1627,7 @@
 
 def test_extern_python_bogus_name():
     ffi = FFI()
-    ffi.cdef("int abc;")
+    ffi.cdef("extern int abc;")
     lib = verify(ffi, 'test_extern_python_bogus_name', "int abc;")
     def fn():
         pass
@@ -1787,8 +1788,8 @@
     ffi.cdef("""
         extern "Python" int __stdcall foo(int);
         extern "Python" int WINAPI bar(int);
-        int (__stdcall * mycb1)(int);
-        int indirect_call(int);
+        static int (__stdcall * mycb1)(int);
+        static int indirect_call(int);
     """)
     lib = verify(ffi, 'test_extern_python_stdcall', """
         #ifndef _MSC_VER
@@ -1856,7 +1857,7 @@
 
 def test_introspect_global_var():
     ffi = FFI()
-    ffi.cdef("float g1;")
+    ffi.cdef("extern float g1;")
     lib = verify(ffi, 'test_introspect_global_var', """
         float g1;
     """)
@@ -1867,7 +1868,7 @@
 
 def test_introspect_global_var_array():
     ffi = FFI()
-    ffi.cdef("float g1[100];")
+    ffi.cdef("extern float g1[100];")
     lib = verify(ffi, 'test_introspect_global_var_array', """
         float g1[100];
     """)
@@ -2039,7 +2040,7 @@
     ffi.cdef("float _Complex f1(float a, float b);");
     lib = verify(ffi, "test_function_returns_float_complex", """
         #include <complex.h>
-        static float _Complex f1(float a, float b) { return a + I*2.0*b; }
+        static float _Complex f1(float a, float b) { return a + I*2.0f*b; }
     """, no_cpp=True)    # <complex.h> fails on some systems with C++
     result = lib.f1(1.25, 5.1)
     assert type(result) == complex
@@ -2090,7 +2091,7 @@
     ffi = FFI()
     ffi.cdef("""
         typedef int foo_t[...], bar_t[...];
-        int gv[...];
+        extern int gv[...];
         typedef int mat_t[...][...];
         typedef int vmat_t[][...];
         """)
diff --git a/extra_tests/cffi_tests/cffi1/test_verify1.py b/extra_tests/cffi_tests/cffi1/test_verify1.py
--- a/extra_tests/cffi_tests/cffi1/test_verify1.py
+++ b/extra_tests/cffi_tests/cffi1/test_verify1.py
@@ -5,7 +5,7 @@
 from cffi import CDefError
 from cffi import recompiler
 from extra_tests.cffi_tests.support import *
-from extra_tests.cffi_tests.support import _verify
+from extra_tests.cffi_tests.support import _verify, extra_compile_args
 import _cffi_backend
 
 lib_m = ['m']
@@ -14,18 +14,6 @@
     import distutils.ccompiler
     if distutils.ccompiler.get_default_compiler() == 'msvc':
         lib_m = ['msvcrt']
-    extra_compile_args = []      # no obvious -Werror equivalent on MSVC
-else:
-    if (sys.platform == 'darwin' and
-          [int(x) for x in os.uname()[2].split('.')] >= [11, 0, 0]):
-        # assume a standard clang or gcc
-        extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion']
-        # special things for clang
-        extra_compile_args.append('-Qunused-arguments')
-    else:
-        # assume a standard gcc
-        extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion',
-                              '-Wno-unused-parameter']
 
 class FFI(FFI):
     error = _cffi_backend.FFI.error
@@ -268,7 +256,7 @@
 def test_var_signed_integer_types():
     ffi = FFI()
     lst = all_signed_integer_types(ffi)
-    csource = "\n".join(["%s somevar_%s;" % (tp, tp.replace(' ', '_'))
+    csource = "\n".join(["static %s somevar_%s;" % (tp, tp.replace(' ', '_'))
                          for tp in lst])
     ffi.cdef(csource)
     lib = ffi.verify(csource)
@@ -287,7 +275,7 @@
 def test_var_unsigned_integer_types():
     ffi = FFI()
     lst = all_unsigned_integer_types(ffi)
-    csource = "\n".join(["%s somevar_%s;" % (tp, tp.replace(' ', '_'))
+    csource = "\n".join(["static %s somevar_%s;" % (tp, tp.replace(' ', '_'))
                          for tp in lst])
     ffi.cdef(csource)
     lib = ffi.verify(csource)
@@ -791,8 +779,8 @@
 
 def test_access_variable():
     ffi = FFI()
-    ffi.cdef("int foo(void);\n"
-             "int somenumber;")
+    ffi.cdef("static int foo(void);\n"
+             "static int somenumber;")
     lib = ffi.verify("""
         static int somenumber = 2;
         static int foo(void) {
@@ -809,7 +797,7 @@
 def test_access_address_of_variable():
     # access the address of 'somenumber': need a trick
     ffi = FFI()
-    ffi.cdef("int somenumber; static int *const somenumberptr;")
+    ffi.cdef("static int somenumber; static int *const somenumberptr;")
     lib = ffi.verify("""
         static int somenumber = 2;
         #define somenumberptr (&somenumber)
@@ -821,8 +809,8 @@
 
 def test_access_array_variable(length=5):
     ffi = FFI()
-    ffi.cdef("int foo(int);\n"
-             "int somenumber[%s];" % (length,))
+    ffi.cdef("static int foo(int);\n"
+             "static int somenumber[%s];" % (length,))
     lib = ffi.verify("""
         static int somenumber[] = {2, 2, 3, 4, 5};
         static int foo(int i) {
@@ -853,8 +841,8 @@
 def test_access_struct_variable():
     ffi = FFI()
     ffi.cdef("struct foo { int x; ...; };\n"
-             "int foo(int);\n"
-             "struct foo stuff;")
+             "static int foo(int);\n"
+             "static struct foo stuff;")
     lib = ffi.verify("""
         struct foo { int x, y, z; };
         static struct foo stuff = {2, 5, 8};
@@ -878,9 +866,9 @@
 
 def test_access_callback():
     ffi = FFI()
-    ffi.cdef("int (*cb)(int);\n"
-             "int foo(int);\n"
-             "void reset_cb(void);")
+    ffi.cdef("static int (*cb)(int);\n"
+             "static int foo(int);\n"
+             "static void reset_cb(void);")
     lib = ffi.verify("""
         static int g(int x) { return x * 7; }
         static int (*cb)(int);
@@ -896,9 +884,9 @@
 def test_access_callback_function_typedef():
     ffi = FFI()
     ffi.cdef("typedef int mycallback_t(int);\n"
-             "mycallback_t *cb;\n"
-             "int foo(int);\n"
-             "void reset_cb(void);")
+             "static mycallback_t *cb;\n"
+             "static int foo(int);\n"
+             "static void reset_cb(void);")
     lib = ffi.verify("""
         static int g(int x) { return x * 7; }
         static int (*cb)(int);
@@ -1039,7 +1027,7 @@
 def test_autofilled_struct_as_argument_dynamic():
     ffi = FFI()
     ffi.cdef("struct foo_s { long a; ...; };\n"
-             "int (*foo)(struct foo_s);")
+             "static int (*foo)(struct foo_s);")
     lib = ffi.verify("""
         struct foo_s {
             double b;
@@ -1048,7 +1036,7 @@
         int foo1(struct foo_s s) {
             return (int)s.a - (int)s.b;
         }
-        int (*foo)(struct foo_s s) = &foo1;
+        static int (*foo)(struct foo_s s) = &foo1;
     """)
     e = py.test.raises(NotImplementedError, lib.foo, "?")
     msg = ("ctype 'struct foo_s' not supported as argument.  It is a struct "
@@ -1424,7 +1412,7 @@
         py.test.skip("_Bool not in MSVC")
     ffi = FFI()
     ffi.cdef("struct foo_s { _Bool x; };"
-             "_Bool foo(_Bool); _Bool (*foop)(_Bool);")
+             "_Bool foo(_Bool); static _Bool (*foop)(_Bool);")
     lib = ffi.verify("""
         struct foo_s { _Bool x; };
         int foo(int arg) {
@@ -1433,7 +1421,7 @@
         _Bool _foofunc(_Bool x) {
             return !x;
         }
-        _Bool (*foop)(_Bool) = _foofunc;
+        static _Bool (*foop)(_Bool) = _foofunc;
     """)
     p = ffi.new("struct foo_s *")
     p.x = 1
@@ -1618,7 +1606,7 @@
 
 def test_FILE_stored_explicitly():
     ffi = FFI()
-    ffi.cdef("int myprintf11(const char *, int); FILE *myfile;")
+    ffi.cdef("int myprintf11(const char *, int); extern FILE *myfile;")
     lib = ffi.verify("""
         #include <stdio.h>
         FILE *myfile;
@@ -1644,13 +1632,13 @@
 
 def test_global_array_with_missing_length():
     ffi = FFI()
-    ffi.cdef("int fooarray[];")
+    ffi.cdef("extern int fooarray[];")
     lib = ffi.verify("int fooarray[50];")
     assert repr(lib.fooarray).startswith("<cdata 'int *'")
 
 def test_global_array_with_dotdotdot_length():
     ffi = FFI()
-    ffi.cdef("int fooarray[...];")
+    ffi.cdef("extern int fooarray[...];")
     lib = ffi.verify("int fooarray[50];")
     assert repr(lib.fooarray).startswith("<cdata 'int[50]'")
 
@@ -1658,7 +1646,7 @@
     py.test.xfail("was detected only because 23 bytes cannot be divided by 4; "
                   "redo more generally")
     ffi = FFI()
-    ffi.cdef("int fooarray[...];")
+    ffi.cdef("extern int fooarray[...];")
     py.test.raises(VerificationError, ffi.verify, "char fooarray[23];")
 
 def test_struct_containing_struct():
@@ -1779,7 +1767,7 @@
 def test_callback_indirection():
     ffi = FFI()
     ffi.cdef("""
-        int (*python_callback)(int how_many, int *values);
+        static int (*python_callback)(int how_many, int *values);
         int (*const c_callback)(int,...);   /* pass this ptr to C routines */
         int some_c_function(int(*cb)(int,...));
     """)
@@ -1913,23 +1901,23 @@
 
 def test_bug_const_char_ptr_array_1():
     ffi = FFI()
-    ffi.cdef("""const char *a[...];""")
+    ffi.cdef("""extern const char *a[...];""")
     lib = ffi.verify("""const char *a[5];""")
     assert repr(ffi.typeof(lib.a)) == "<ctype 'char *[5]'>"
 
 def test_bug_const_char_ptr_array_2():
     ffi = FFI()
-    ffi.cdef("""const int a[];""")
+    ffi.cdef("""extern const int a[];""")
     lib = ffi.verify("""const int a[5];""")
     assert repr(ffi.typeof(lib.a)) == "<ctype 'int *'>"
 
 def _test_various_calls(force_libffi):
     cdef_source = """
-    int xvalue;
-    long long ivalue, rvalue;
-    float fvalue;
-    double dvalue;
-    long double Dvalue;
+    extern int xvalue;
+    extern long long ivalue, rvalue;
+    extern float fvalue;
+    extern double dvalue;
+    extern long double Dvalue;
     signed char tf_bb(signed char x, signed char c);
     unsigned char tf_bB(signed char x, unsigned char c);
     short tf_bh(signed char x, short c);
@@ -2112,7 +2100,7 @@
     old = sys.getdlopenflags()
     try:
         ffi1 = FFI()
-        ffi1.cdef("int foo_verify_dlopen_flags_1;")
+        ffi1.cdef("extern int foo_verify_dlopen_flags_1;")
         sys.setdlopenflags(ffi1.RTLD_GLOBAL | ffi1.RTLD_NOW)
         lib1 = ffi1.verify("int foo_verify_dlopen_flags_1;")
     finally:
@@ -2253,7 +2241,7 @@
 
 def test_macro_var():
     ffi = FFI()
-    ffi.cdef("int myarray[50], my_value;")
+    ffi.cdef("extern int myarray[50], my_value;")
     lib = ffi.verify("""
         int myarray[50];
         int *get_my_value(void) {
diff --git a/extra_tests/cffi_tests/embedding/add1.py b/extra_tests/cffi_tests/embedding/add1.py
--- a/extra_tests/cffi_tests/embedding/add1.py
+++ b/extra_tests/cffi_tests/embedding/add1.py
@@ -12,7 +12,11 @@
     sys.stdout.write("preparing")
     for i in range(3):
         sys.stdout.flush()
-        time.sleep(0.2)
+        # Windows: sometimes time.sleep() doesn't sleep at all.
+        # This appears to occur on recent versions of python only.
+        t_end = time.time() + 0.19
+        while time.time() < t_end:
+            time.sleep(0.2)
         sys.stdout.write(".")
     sys.stdout.write("\n")
 
diff --git a/extra_tests/cffi_tests/embedding/add_recursive.py b/extra_tests/cffi_tests/embedding/add_recursive.py
--- a/extra_tests/cffi_tests/embedding/add_recursive.py
+++ b/extra_tests/cffi_tests/embedding/add_recursive.py
@@ -4,7 +4,7 @@
 ffi = cffi.FFI()
 
 ffi.embedding_api("""
-    int (*my_callback)(int);
+    extern int (*my_callback)(int);
     int add_rec(int, int);
 """)
 
diff --git a/extra_tests/cffi_tests/embedding/test_thread.py b/extra_tests/cffi_tests/embedding/test_thread.py
--- a/extra_tests/cffi_tests/embedding/test_thread.py
+++ b/extra_tests/cffi_tests/embedding/test_thread.py
@@ -22,17 +22,21 @@
         add1_cffi = self.prepare_module('add1')
         add2_cffi = self.prepare_module('add2')
         self.compile('thread2-test', [add1_cffi, add2_cffi], threads=True)
-        output = self.execute('thread2-test')
-        output = self._take_out(output, "preparing")
-        output = self._take_out(output, ".")
-        output = self._take_out(output, ".")
-        # at least the 3rd dot should be after everything from ADD2
-        assert output == ("starting\n"
-                          "prepADD2\n"
-                          "adding 1000 and 200 and 30\n"
-                          ".\n"
-                          "adding 40 and 2\n"
-                          "done\n")
+        for i in range(3):
+            output = self.execute('thread2-test')
+            print('='*79)
+            print(output)
+            print('='*79)
+            output = self._take_out(output, "preparing")
+            output = self._take_out(output, ".")
+            output = self._take_out(output, ".")
+            # at least the 3rd dot should be after everything from ADD2
+            assert output == ("starting\n"
+                              "prepADD2\n"
+                              "adding 1000 and 200 and 30\n"
+                              ".\n"
+                              "adding 40 and 2\n"
+                              "done\n")
 
     def test_alt_issue(self):
         add1_cffi = self.prepare_module('add1')
diff --git a/extra_tests/cffi_tests/support.py b/extra_tests/cffi_tests/support.py
--- a/extra_tests/cffi_tests/support.py
+++ b/extra_tests/cffi_tests/support.py
@@ -1,5 +1,5 @@
 # Generated by pypy/tool/import_cffi.py
-import sys
+import sys, os
 
 if sys.version_info < (3,):
     __all__ = ['u']
@@ -87,3 +87,24 @@
         if not name.startswith('_') and not hasattr(module.ffi, name):
             setattr(ffi, name, NotImplemented)
     return module.lib
+
+
+# For testing, we call gcc with "-Werror".  This is fragile because newer
+# versions of gcc are always better at producing warnings, particularly for
+# auto-generated code.  We need here to adapt and silence them as needed.
+
+if sys.platform == 'win32':
+    extra_compile_args = []      # no obvious -Werror equivalent on MSVC
+else:
+    if (sys.platform == 'darwin' and
+          [int(x) for x in os.uname()[2].split('.')] >= [11, 0, 0]):
+        # assume a standard clang or gcc
+        extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion',
+                              '-Wno-unreachable-code']
+        # special things for clang
+        extra_compile_args.append('-Qunused-arguments')
+    else:
+        # assume a standard gcc
+        extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion',
+                              '-Wno-unused-parameter',
+                              '-Wno-unreachable-code']
diff --git a/extra_tests/cffi_tests/test_version.py b/extra_tests/cffi_tests/test_version.py
--- a/extra_tests/cffi_tests/test_version.py
+++ b/extra_tests/cffi_tests/test_version.py
@@ -1,7 +1,7 @@
 from email.parser import Parser
 
 import py
-import urllib2
+from urllib.request import urlopen
 
 import cffi
 import pypy
@@ -14,9 +14,8 @@
 
 def test_pycparser_version():
     url = 'https://raw.githubusercontent.com/eliben/pycparser/master/pycparser/__init__.py'
-    source = urllib2.urlopen(url).read()
+    source = urlopen(url).read().decode('utf8')
     dest = py.path.local(__file__).join('..', '..', '..', 'lib_pypy', 'cffi',
                                         '_pycparser', '__init__.py').read()
     # if this fails, the vendored pycparser is not the latest version
     assert source.strip() == dest.strip()
-    
diff --git a/pypy/module/_asyncio/test/test_asyncio.py b/extra_tests/test_asyncio.py
rename from pypy/module/_asyncio/test/test_asyncio.py
rename to extra_tests/test_asyncio.py
--- a/pypy/module/_asyncio/test/test_asyncio.py
+++ b/extra_tests/test_asyncio.py
@@ -1,116 +1,96 @@
 import sys
 
-class AppTestAsyncIO(object):
-    """These tests are based on the async-await syntax of Python 3.5."""
-    
-    spaceconfig = dict(usemodules=["select","_socket","thread","signal",
-                                   "struct","_multiprocessing","array",
-                                   "_posixsubprocess",
-                                   "unicodedata"])
-    if sys.platform == 'win32':
-        pass
-    else:
-        spaceconfig['usemodules'].append('fcntl')
-    
-    def test_gil_issue(self):
-        # the problem occured at await asyncio.open_connection
-        # after calling run_until_complete
-        """
 import encodings.idna
 import asyncio
 
-async def f():
-    reader, writer = await asyncio.open_connection('example.com', 80)
-    writer.close()
+def test_async_gil_issue():
+    async def f():
+        reader, writer = await asyncio.open_connection('example.com', 80)
+        writer.close()
 
-loop = asyncio.get_event_loop()
-loop.run_until_complete(f())
-        """
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(f())
     
-    def test_async_for(self):
-        # tests if async for receives all stores values in the right order
-        # and if the correct methods __aiter__ and __anext__ get called
-        # and if the end results of run_until_complete are None (in a tuple)
-        """
-import asyncio
+def test_async_for():
+    # tests if async for receives all stores values in the right order
+    # and if the correct methods __aiter__ and __anext__ get called
+    # and if the end results of run_until_complete are None (in a tuple)
+    import asyncio
 
-class AsyncIter:
-    def __init__(self):
-        self._data = list(range(5))
-        self._index = 0
+    class AsyncIter:
+        def __init__(self):
+            self._data = list(range(5))
+            self._index = 0
+        
+        async def __aiter__(self):
+            return self
+        
+        async def __anext__(self):
+            while self._index < 5:
+                await asyncio.sleep(1)
+                self._index += 1
+                return self._data[self._index-1]
+            raise StopAsyncIteration
+
+    class Corotest(object):
+        def __init__(self):
+            self.res = "-"
+        
+        async def do_loop(self):
+            async for x in AsyncIter():
+                self.res += str(x)
+                self.res += "-"
+
+    cor = Corotest()
+    loop = asyncio.get_event_loop()
+    futures = [asyncio.ensure_future(cor.do_loop()), asyncio.ensure_future(cor.do_loop())]
+    taskres = loop.run_until_complete(asyncio.wait(futures))
+    assert cor.res.count('0') == 2
+    assert cor.res.count('1') == 2
+    assert cor.res.count('2') == 2
+    assert cor.res.count('3') == 2
+    assert cor.res.count('4') == 2
+    assert cor.res.find("0") < cor.res.find("1")
+    assert cor.res.find("1") < cor.res.find("2")
+    assert cor.res.find("2") < cor.res.find("3")
+    assert cor.res.find("3") < cor.res.find("4")
+    assert isinstance(taskres, tuple)
+    assert len(taskres) == 2
+    assert "result=None" in repr(taskres[0].pop())
+    assert "result=None" in repr(taskres[0].pop())
     
-    async def __aiter__(self):
-        return self
-    
-    async def __anext__(self):
-        while self._index < 5:
-            await asyncio.sleep(1)
-            self._index += 1
-            return self._data[self._index-1]
-        raise StopAsyncIteration
+def test_asynchronous_context_managers():
+    # it is important that "releasing lock A" happens before "holding lock B"
+    # or the other way around, but it is not allowed that both coroutines
+    # hold the lock at the same time
+    import encodings.idna
+    import asyncio
 
-class Corotest(object):
-    def __init__(self):
-        self.res = "-"
-    
-    async def do_loop(self):
-        async for x in AsyncIter():
-            self.res += str(x)
-            self.res += "-"
+    class Corotest(object):
+        def __init__(self):
+            self.res = "-"
+        
+        async def coro(self, name, lock):
+            self.res += ' coro {}: waiting for lock -'.format(name)
+            async with lock:
+                self.res += ' coro {}: holding the lock -'.format(name)
+                await asyncio.sleep(1)
+                self.res += ' coro {}: releasing the lock -'.format(name)
 
-cor = Corotest()
-loop = asyncio.get_event_loop()
-futures = [asyncio.ensure_future(cor.do_loop()), asyncio.ensure_future(cor.do_loop())]
-taskres = loop.run_until_complete(asyncio.wait(futures))
-assert cor.res.count('0') == 2
-assert cor.res.count('1') == 2
-assert cor.res.count('2') == 2
-assert cor.res.count('3') == 2
-assert cor.res.count('4') == 2
-assert cor.res.find("0") < cor.res.find("1")
-assert cor.res.find("1") < cor.res.find("2")
-assert cor.res.find("2") < cor.res.find("3")
-assert cor.res.find("3") < cor.res.find("4")
-assert isinstance(taskres, tuple)
-assert len(taskres) == 2
-assert "result=None" in repr(taskres[0].pop())
-assert "result=None" in repr(taskres[0].pop())
-        """
-    
-    def test_asynchronous_context_managers(self):
-        # it is important that "releasing lock A" happens before "holding lock B"
-        # or the other way around, but it is not allowed that both coroutines
-        # hold the lock at the same time
-        """
-import encodings.idna
-import asyncio
+    cor = Corotest()
+    loop = asyncio.get_event_loop()
+    lock = asyncio.Lock()
+    coros = asyncio.gather(cor.coro(1, lock), cor.coro(2, lock))
+    try:
+        loop.run_until_complete(coros)
+    finally:
+        loop.close()
 
-class Corotest(object):
-    def __init__(self):
-        self.res = "-"
-    
-    async def coro(self, name, lock):
-        self.res += ' coro {}: waiting for lock -'.format(name)
-        async with lock:
-            self.res += ' coro {}: holding the lock -'.format(name)
-            await asyncio.sleep(1)
-            self.res += ' coro {}: releasing the lock -'.format(name)
-
-cor = Corotest()
-loop = asyncio.get_event_loop()
-lock = asyncio.Lock()
-coros = asyncio.gather(cor.coro(1, lock), cor.coro(2, lock))
-try:
-    loop.run_until_complete(coros)
-finally:
-    loop.close()
-
-assert "coro 1: waiting for lock" in cor.res
-assert "coro 1: holding the lock" in cor.res
-assert "coro 1: releasing the lock" in cor.res
-assert "coro 2: waiting for lock" in cor.res
-assert "coro 2: holding the lock" in cor.res
-assert "coro 2: releasing the lock" in cor.res
-assert cor.res.find("coro 1: releasing the lock") < cor.res.find("coro 2: holding the lock") or \
-cor.res.find("coro 2: releasing the lock") < cor.res.find("coro 1: holding the lock")
-        """
+    assert "coro 1: waiting for lock" in cor.res
+    assert "coro 1: holding the lock" in cor.res
+    assert "coro 1: releasing the lock" in cor.res
+    assert "coro 2: waiting for lock" in cor.res
+    assert "coro 2: holding the lock" in cor.res
+    assert "coro 2: releasing the lock" in cor.res
+    assert cor.res.find("coro 1: releasing the lock") < cor.res.find("coro 2: holding the lock") or \
+    cor.res.find("coro 2: releasing the lock") < cor.res.find("coro 1: holding the lock")
diff --git a/extra_tests/test_pyrepl/infrastructure.py b/extra_tests/test_pyrepl/infrastructure.py
--- a/extra_tests/test_pyrepl/infrastructure.py
+++ b/extra_tests/test_pyrepl/infrastructure.py
@@ -59,6 +59,11 @@
         return Event(*ev)
 
 
+    def getpending(self):
+        """Nothing pending, but do not return None here."""
+        return Event('key', '', b'')
+
+
 class BaseTestReader(Reader):
 
     def get_prompt(self, lineno, cursor_on_line):
diff --git a/extra_tests/test_pyrepl/test_readline.py b/extra_tests/test_pyrepl/test_readline.py
--- a/extra_tests/test_pyrepl/test_readline.py
+++ b/extra_tests/test_pyrepl/test_readline.py
@@ -15,8 +15,7 @@
     os.write(master, b'input\n')
 
     with sane_term():
-        result = readline_wrapper.get_reader().readline()
-    #result = readline_wrapper.raw_input('prompt:')
+        result = readline_wrapper.raw_input('prompt:')
     assert result == 'input'
     # A bytes string on python2, a unicode string on python3.
     assert isinstance(result, str)
diff --git a/extra_tests/test_pyrepl/test_wishes.py b/extra_tests/test_pyrepl/test_wishes.py
--- a/extra_tests/test_pyrepl/test_wishes.py
+++ b/extra_tests/test_pyrepl/test_wishes.py
@@ -27,5 +27,5 @@
     read_spec([
         (('digit-arg', '3'),      ['']),
         (('quoted-insert', None), ['']),
-        (('self-insert', '\033'), ['^[^[^[']),
+        (('key', '\033'), ['^[^[^[']),
         (('accept', None),        None)])
diff --git a/extra_tests/test_semlock.py b/extra_tests/test_semlock.py
--- a/extra_tests/test_semlock.py
+++ b/extra_tests/test_semlock.py
@@ -1,6 +1,6 @@
 from _multiprocessing import SemLock
 from threading import Thread
-import thread
+import _thread
 import time
 
 
@@ -23,7 +23,7 @@
         for t in threads:
             try:
                 t.start()
-            except thread.error:
+            except _thread.error:
                 # too many threads for this system
                 t.started = False
                 n_started -= 1
diff --git a/lib-python/3/_osx_support.py b/lib-python/3/_osx_support.py
--- a/lib-python/3/_osx_support.py
+++ b/lib-python/3/_osx_support.py
@@ -109,7 +109,7 @@
             # else: fall back to the default behaviour
     if not _SYSTEM_VERSION:
         # minimum supported MACOSX_DEPLOYMENT_TARGET version
-        return '10.14'
+        return '10.7'
     return _SYSTEM_VERSION
 
 def _remove_original_values(_config_vars):
diff --git a/lib-python/3/datetime.py b/lib-python/3/datetime.py
--- a/lib-python/3/datetime.py
+++ b/lib-python/3/datetime.py
@@ -609,31 +609,31 @@
         if isinstance(other, timedelta):
             return self._cmp(other) == 0
         else:
-            return False
+            return NotImplemented
 
     def __le__(self, other):
         if isinstance(other, timedelta):
             return self._cmp(other) <= 0
         else:
-            _cmperror(self, other)
+            return NotImplemented
 
     def __lt__(self, other):
         if isinstance(other, timedelta):
             return self._cmp(other) < 0
         else:
-            _cmperror(self, other)
+            return NotImplemented
 
     def __ge__(self, other):
         if isinstance(other, timedelta):
             return self._cmp(other) >= 0
         else:
-            _cmperror(self, other)
+            return NotImplemented
 
     def __gt__(self, other):
         if isinstance(other, timedelta):
             return self._cmp(other) > 0
         else:
-            _cmperror(self, other)
+            return NotImplemented
 
     def _cmp(self, other):
         assert isinstance(other, timedelta)
@@ -1144,31 +1144,31 @@
         if isinstance(other, time):
             return self._cmp(other, allow_mixed=True) == 0
         else:
-            return False
+            return NotImplemented
 
     def __le__(self, other):
         if isinstance(other, time):
             return self._cmp(other) <= 0
         else:
-            _cmperror(self, other)
+            return NotImplemented
 
     def __lt__(self, other):
         if isinstance(other, time):
             return self._cmp(other) < 0
         else:
-            _cmperror(self, other)
+            return NotImplemented
 
     def __ge__(self, other):
         if isinstance(other, time):
             return self._cmp(other) >= 0
         else:
-            _cmperror(self, other)
+            return NotImplemented
 
     def __gt__(self, other):
         if isinstance(other, time):
             return self._cmp(other) > 0
         else:
-            _cmperror(self, other)
+            return NotImplemented
 
     def _cmp(self, other, allow_mixed=False):
         assert isinstance(other, time)
@@ -2048,9 +2048,9 @@
         return (self._offset, self._name)
 
     def __eq__(self, other):
-        if type(other) != timezone:
-            return False
-        return self._offset == other._offset
+        if isinstance(other, timezone):
+            return self._offset == other._offset
+        return NotImplemented
 
     def __hash__(self):
         return hash(self._offset)
diff --git a/lib-python/3/distutils/command/build_ext.py b/lib-python/3/distutils/command/build_ext.py
--- a/lib-python/3/distutils/command/build_ext.py
+++ b/lib-python/3/distutils/command/build_ext.py
@@ -28,11 +28,6 @@
     from distutils.ccompiler import show_compilers
     show_compilers()
 
-def _get_c_extension_suffix():
-    import importlib
-    suffixes = importlib.machinery.EXTENSION_SUFFIXES
-    return suffixes[0] if suffixes else None
-
 
 class build_ext(Command):
 
@@ -686,15 +681,7 @@
         """
         from distutils.sysconfig import get_config_var
         ext_path = ext_name.split('.')
-        # PyPy tweak: first try to get the C extension suffix from
-        # 'imp'.  If it fails we fall back to the 'SO' config var, like
-        # the previous version of this code did.  This should work for
-        # CPython too.  The point is that on PyPy with cpyext, the
-        # config var 'SO' is just ".so" but we want to return
-        # ".pypy-VERSION.so" instead.
-        ext_suffix = _get_c_extension_suffix()
-        if ext_suffix is None:
-            ext_suffix = get_config_var('EXT_SUFFIX')     # fall-back
+        ext_suffix = get_config_var('EXT_SUFFIX')
         return os.path.join(*ext_path) + ext_suffix
 
     def get_export_symbols(self, ext):
diff --git a/lib-python/3/ensurepip/__init__.py b/lib-python/3/ensurepip/__init__.py
--- a/lib-python/3/ensurepip/__init__.py
+++ b/lib-python/3/ensurepip/__init__.py
@@ -8,9 +8,9 @@
 __all__ = ["version", "bootstrap"]
 
 
-_SETUPTOOLS_VERSION = "40.6.2"
+_SETUPTOOLS_VERSION = "41.2.0"
 
-_PIP_VERSION = "18.1"
+_PIP_VERSION = "19.2.3"
 
 _PROJECTS = [
     ("setuptools", _SETUPTOOLS_VERSION),
diff --git a/lib-python/3/ensurepip/_bundled/pip-18.1-py2.py3-none-any.whl b/lib-python/3/ensurepip/_bundled/pip-18.1-py2.py3-none-any.whl
deleted file mode 100644
index c3c146f6da272399a19fd6e21364a4f45066cec7..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch

[cut]

diff --git a/lib-python/3/ensurepip/_bundled/pip-19.2.3-py2.py3-none-any.whl b/lib-python/3/ensurepip/_bundled/pip-19.2.3-py2.py3-none-any.whl
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..8118df8ac1940f8c6cb410fbc18e5fae59872b95
GIT binary patch

[cut]
diff --git a/lib-python/3/ensurepip/_bundled/setuptools-40.6.2-py2.py3-none-any.whl b/lib-python/3/ensurepip/_bundled/setuptools-40.6.2-py2.py3-none-any.whl
deleted file mode 100644
index 4c8a619571d1e50ae8b6b961a4bebc32c90694e1..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch

[cut]

diff --git a/lib-python/3/ensurepip/_bundled/setuptools-41.2.0-py2.py3-none-any.whl b/lib-python/3/ensurepip/_bundled/setuptools-41.2.0-py2.py3-none-any.whl
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..82df6f63f4ee97380af0a29d8825ae775333b86d
GIT binary patch

[cut]

diff --git a/lib-python/3/sysconfig.py b/lib-python/3/sysconfig.py
--- a/lib-python/3/sysconfig.py
+++ b/lib-python/3/sysconfig.py
@@ -450,6 +450,10 @@
     vars['EXE'] = '.exe'
     vars['VERSION'] = _PY_VERSION_SHORT_NO_DOT
     vars['BINDIR'] = os.path.dirname(_safe_realpath(sys.executable))
+    # pypy: give us control over the ABI tag in a wheel name
+    import _imp
+    so_ext = _imp.extension_suffixes()[0]
+    vars['SOABI']= '-'.join(so_ext.split('.')[1].split('-')[:2])
 
 #
 # public APIs
diff --git a/lib-python/3/test/test_asyncio/test_events.py b/lib-python/3/test/test_asyncio/test_events.py
--- a/lib-python/3/test/test_asyncio/test_events.py
+++ b/lib-python/3/test/test_asyncio/test_events.py
@@ -932,9 +932,14 @@
         server = self.loop.run_until_complete(f)
         self.assertEqual(len(server.sockets), 1)
         sock = server.sockets[0]
-        self.assertFalse(
-            sock.getsockopt(
-                socket.SOL_SOCKET, socket.SO_REUSEPORT))
+        try:
+            self.assertFalse(
+                sock.getsockopt(
+                    socket.SOL_SOCKET, socket.SO_REUSEPORT))
+        except OSError:
+            # SO_REUSEPORT is not actually supported, bail!
+            server.close()
+            return
         server.close()
 
         test_utils.run_briefly(self.loop)
diff --git a/lib-python/3/test/test_bdb.py b/lib-python/3/test/test_bdb.py
--- a/lib-python/3/test/test_bdb.py
+++ b/lib-python/3/test/test_bdb.py
@@ -726,7 +726,7 @@
                 ('line', 2, 'tfunc_import'), ('step', ),
                 ('line', 3, 'tfunc_import'), ('quit', ),
             ]
-            skip = ('importlib*', TEST_MODULE)
+            skip = ('importlib*', '_structseq', TEST_MODULE)
             with TracerRun(self, skip=skip) as tracer:
                 tracer.runcall(tfunc_import)
 
diff --git a/lib-python/3/test/test_dis.py b/lib-python/3/test/test_dis.py
--- a/lib-python/3/test/test_dis.py
+++ b/lib-python/3/test/test_dis.py
@@ -262,7 +262,7 @@
              20 RETURN_VALUE
 """
 
-# XXX: change for PyPy?
+# changed for PyPy
 dis_traceback = """\
 %3d           0 SETUP_EXCEPT            12 (to 14)
 
@@ -280,18 +280,18 @@
              22 POP_TOP
              24 STORE_FAST               0 (e)
              26 POP_TOP
-             28 SETUP_FINALLY           12 (to 42)
+             28 SETUP_FINALLY           10 (to 40)
 
 %3d          30 LOAD_FAST                0 (e)
              32 LOAD_ATTR                1 (__traceback__)
              34 STORE_FAST               1 (tb)
              36 POP_BLOCK
-             38 POP_EXCEPT
-             40 LOAD_CONST               0 (None)
-        >>   42 LOAD_CONST               0 (None)
-             44 STORE_FAST               0 (e)
-             46 DELETE_FAST              0 (e)
-             48 END_FINALLY
+             38 LOAD_CONST               0 (None)
+        >>   40 LOAD_CONST               0 (None)
+             42 STORE_FAST               0 (e)
+             44 DELETE_FAST              0 (e)
+             46 END_FINALLY
+             48 POP_EXCEPT
              50 JUMP_FORWARD             2 (to 54)
         >>   52 END_FINALLY
 
@@ -686,9 +686,9 @@
 # End fodder for opinfo generation tests
 expected_outer_line = 1
 _line_offset = outer.__code__.co_firstlineno - 1
-code_object_f = outer.__code__.co_consts[3]
+code_object_f = outer.__code__.co_consts[2]
 expected_f_line = code_object_f.co_firstlineno - _line_offset
-code_object_inner = code_object_f.co_consts[3]
+code_object_inner = code_object_f.co_consts[2]
 expected_inner_line = code_object_inner.co_firstlineno - _line_offset
 expected_jumpy_line = 1
 
@@ -713,22 +713,22 @@
 
 Instruction = dis.Instruction
 expected_opinfo_outer = [
-  Instruction(opname='LOAD_CONST', opcode=100, arg=8, argval=(3, 4), argrepr='(3, 4)', offset=0, starts_line=2, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=1, argval=(3, 4), argrepr='(3, 4)', offset=0, starts_line=2, is_jump_target=False),
   Instruction(opname='LOAD_CLOSURE', opcode=135, arg=0, argval='a', argrepr='a', offset=2, starts_line=None, is_jump_target=False),
   Instruction(opname='LOAD_CLOSURE', opcode=135, arg=1, argval='b', argrepr='b', offset=4, starts_line=None, is_jump_target=False),
   Instruction(opname='BUILD_TUPLE', opcode=102, arg=2, argval=2, argrepr='', offset=6, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=code_object_f, argrepr=repr(code_object_f), offset=8, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='outer.<locals>.f', argrepr="'outer.<locals>.f'", offset=10, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=code_object_f, argrepr=repr(code_object_f), offset=8, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval='outer.<locals>.f', argrepr="'outer.<locals>.f'", offset=10, starts_line=None, is_jump_target=False),
   Instruction(opname='MAKE_FUNCTION', opcode=132, arg=9, argval=9, argrepr='', offset=12, starts_line=None, is_jump_target=False),
   Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='f', argrepr='f', offset=14, starts_line=None, is_jump_target=False),
   Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=16, starts_line=7, is_jump_target=False),
   Instruction(opname='LOAD_DEREF', opcode=136, arg=0, argval='a', argrepr='a', offset=18, starts_line=None, is_jump_target=False),
   Instruction(opname='LOAD_DEREF', opcode=136, arg=1, argval='b', argrepr='b', offset=20, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval='', argrepr="''", offset=22, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval=1, argrepr='1', offset=24, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='', argrepr="''", offset=22, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=24, starts_line=None, is_jump_target=False),
   Instruction(opname='BUILD_LIST', opcode=103, arg=0, argval=0, argrepr='', offset=26, starts_line=None, is_jump_target=False),
   Instruction(opname='BUILD_MAP', opcode=105, arg=0, argval=0, argrepr='', offset=28, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=7, argval='Hello world!', argrepr="'Hello world!'", offset=30, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval='Hello world!', argrepr="'Hello world!'", offset=30, starts_line=None, is_jump_target=False),
   Instruction(opname='CALL_FUNCTION', opcode=131, arg=7, argval=7, argrepr='', offset=32, starts_line=None, is_jump_target=False),
   Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=34, starts_line=None, is_jump_target=False),
   Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='f', argrepr='f', offset=36, starts_line=8, is_jump_target=False),
@@ -736,14 +736,14 @@
 ]
 
 expected_opinfo_f = [
-  Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=(5, 6), argrepr='(5, 6)', offset=0, starts_line=3, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=1, argval=(5, 6), argrepr='(5, 6)', offset=0, starts_line=3, is_jump_target=False),
   Instruction(opname='LOAD_CLOSURE', opcode=135, arg=2, argval='a', argrepr='a', offset=2, starts_line=None, is_jump_target=False),
   Instruction(opname='LOAD_CLOSURE', opcode=135, arg=3, argval='b', argrepr='b', offset=4, starts_line=None, is_jump_target=False),
   Instruction(opname='LOAD_CLOSURE', opcode=135, arg=0, argval='c', argrepr='c', offset=6, starts_line=None, is_jump_target=False),
   Instruction(opname='LOAD_CLOSURE', opcode=135, arg=1, argval='d', argrepr='d', offset=8, starts_line=None, is_jump_target=False),
   Instruction(opname='BUILD_TUPLE', opcode=102, arg=4, argval=4, argrepr='', offset=10, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=code_object_inner, argrepr=repr(code_object_inner), offset=12, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='outer.<locals>.f.<locals>.inner', argrepr="'outer.<locals>.f.<locals>.inner'", offset=14, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=code_object_inner, argrepr=repr(code_object_inner), offset=12, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval='outer.<locals>.f.<locals>.inner', argrepr="'outer.<locals>.f.<locals>.inner'", offset=14, starts_line=None, is_jump_target=False),
   Instruction(opname='MAKE_FUNCTION', opcode=132, arg=9, argval=9, argrepr='', offset=16, starts_line=None, is_jump_target=False),
   Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='inner', argrepr='inner', offset=18, starts_line=None, is_jump_target=False),
   Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=20, starts_line=5, is_jump_target=False),
diff --git a/lib-python/3/test/test_extcall.py b/lib-python/3/test/test_extcall.py
--- a/lib-python/3/test/test_extcall.py
+++ b/lib-python/3/test/test_extcall.py
@@ -57,7 +57,7 @@
     Traceback (most recent call last):
         ...
     TypeError: ...got multiple values for keyword argument 'a'
-    >>> f(1, 2, a=3, **{'a': 4}, **{'a': 5})
+    >>> f(1, 2, a=3, **{'a': 4}, **{'a': 5})  #doctest: +ELLIPSIS
     Traceback (most recent call last):
         ...
     TypeError: ...got multiple values for keyword argument 'a'
@@ -254,20 +254,21 @@
       ...
     TypeError: h() argument after * must be an iterable, not function
 
-    >>> h(*[1], *h)
+    >>> h(*[1], *h)  #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: h() argument after * must be an iterable, not function
+    TypeError: ...
 
     >>> dir(*h)
     Traceback (most recent call last):
       ...
     TypeError: dir() argument after * must be an iterable, not function
 
-    >>> None(*h)
+    >>> None(**h)  #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: ...argument after * must be an iterable, not function
+    TypeError: ... object argument after ** must be a mapping, \
+not function
 
     >>> h(**h)
     Traceback (most recent call last):
@@ -289,35 +290,20 @@
       ...
     TypeError: h() argument after ** must be a mapping, not list
 
-    >>> h(**{'a': 1}, **h)
+    >>> h(**{'a': 1}, **h)  #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: h() argument after ** must be a mapping, not function
+    TypeError: ...argument after ** must be a mapping, not function
 
-    >>> h(**{'a': 1}, **[])
+    >>> h(**{'a': 1}, **[])  #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: h() argument after ** must be a mapping, not list
+    TypeError: ...argument after ** must be a mapping, not list
 
     >>> dir(**h)
     Traceback (most recent call last):
       ...
-    TypeError: ...argument after * must be an iterable, not function
-
-    >>> None(*h)                               #doctest: +ELLIPSIS
-    Traceback (most recent call last):
-      ...
-    TypeError: ...argument after * must be an iterable, not function
-
-    >>> h(**h)                                 #doctest: +ELLIPSIS
-    Traceback (most recent call last):
-      ...
-    TypeError: ...argument after ** must be a mapping, not function
-
-    >>> dir(**h)                               #doctest: +ELLIPSIS
-    Traceback (most recent call last):
-      ...
-    TypeError: ...argument after ** must be a mapping, not function
+    TypeError: dir() argument after ** must be a mapping, not function
 
     >>> None(**h)                              #doctest: +ELLIPSIS
     Traceback (most recent call last):
diff --git a/lib-python/3/test/test_flufl.py b/lib-python/3/test/test_flufl.py
--- a/lib-python/3/test/test_flufl.py
+++ b/lib-python/3/test/test_flufl.py
@@ -15,7 +15,7 @@
         self.assertEqual(cm.exception.text, '2 != 3\n')
         self.assertEqual(cm.exception.filename, '<FLUFL test>')
         self.assertEqual(cm.exception.lineno, 2)
-        self.assertEqual(cm.exception.offset, 4)
+        self.assertEqual(cm.exception.offset, 2)  # changed in PyPy
 
     def test_guido_as_bdfl(self):
         code = '2 {0} 3'
@@ -26,7 +26,7 @@
         self.assertEqual(cm.exception.text, '2 <> 3\n')
         self.assertEqual(cm.exception.filename, '<FLUFL test>')
         self.assertEqual(cm.exception.lineno, 1)
-        self.assertEqual(cm.exception.offset, 4)
+        self.assertEqual(cm.exception.offset, 2)  # changed in PyPy
 
 
 if __name__ == '__main__':
diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -2,8 +2,18 @@
 import time
 import _thread
 import weakref
-from _pypy_openssl import ffi
-from _pypy_openssl import lib
+
+try:
+    from _pypy_openssl import ffi
+    from _pypy_openssl import lib
+except ImportError as e:
+    import os
+    msg = "\n\nThe _ssl cffi module either doesn't exist or is incompatible with your machine's shared libraries.\n" + \
+          "If you have a compiler installed, you can try to rebuild it by running:\n" + \
+          "cd %s\n" % os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) + \
+          "%s _ssl_build.py\n" % sys.executable
+    raise ImportError(str(e) + msg)
+
 from _cffi_ssl._stdssl.certificate import (_test_decode_cert,
     _decode_certificate, _certificate_to_der)
 from _cffi_ssl._stdssl.utility import (_str_with_len, _bytes_with_len,
diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py
--- a/lib_pypy/_ctypes/array.py
+++ b/lib_pypy/_ctypes/array.py
@@ -275,6 +275,8 @@
         raise TypeError("Can't multiply a ctypes type by a non-integer")
     if length < 0:
         raise ValueError("Array length must be >= 0")
+    if length * base._sizeofinstances() > sys.maxsize:
+        raise OverflowError("array too large")
     key = (base, length)
     try:
         return ARRAY_CACHE[key]
diff --git a/lib_pypy/_curses_build.py b/lib_pypy/_curses_build.py
--- a/lib_pypy/_curses_build.py
+++ b/lib_pypy/_curses_build.py
@@ -1,31 +1,42 @@
+from cffi import FFI, VerificationError
 import os
-from cffi import FFI, VerificationError
 
+version_str = '''
+    static const int NCURSES_VERSION_MAJOR;
+    static const int NCURSES_VERSION_MINOR;
+'''
 
-def find_curses_library():
-    for curses_library in ['ncursesw', 'ncurses']:
+version = (0, 0)
+def find_library(options):
+    global version
+    for library in options:
         ffi = FFI()
-        ffi.set_source("_curses_cffi_check", "", libraries=[curses_library])
+        ffi.cdef(version_str)
+        ffi.set_source("_curses_cffi_check", version_str, libraries=[library])
         try:
             ffi.compile()
+            import _curses_cffi_check
+            lib = _curses_cffi_check.lib
+            version = (lib.NCURSES_VERSION_MAJOR, lib.NCURSES_VERSION_MINOR)
         except VerificationError as e:
             e_last = e
             continue
         else:
-            return curses_library
+            return library
 
-    # If none of the libraries is available, present the user a meaningful
+    # If none of the options is available, present the user a meaningful
     # error message
     raise e_last
 
 def find_curses_include_dirs():
+    if os.path.exists('/usr/include/ncurses'):
+        return ['/usr/include/ncurses']
     if os.path.exists('/usr/include/ncursesw'):
         return ['/usr/include/ncursesw']
     return []
 
 
 ffi = FFI()
-
 ffi.set_source("_curses_cffi", """
 #ifdef __APPLE__
 /* the following define is necessary for OS X 10.6+; without it, the
@@ -72,7 +83,8 @@
 void _m_getsyx(int *yx) {
     getsyx(yx[0], yx[1]);
 }
-""", libraries=[find_curses_library(), 'panel'],
+""", libraries=[find_library(['ncurses', 'ncursesw']),
+                find_library(['panel', 'panelw'])],
      include_dirs=find_curses_include_dirs())
 
 
@@ -84,8 +96,6 @@
 typedef unsigned long... chtype;
 typedef chtype attr_t;
 
-typedef int... wint_t;
-
 typedef struct
 {
     short id;           /* ID to distinguish multiple devices */
@@ -157,11 +167,11 @@
 
 int setupterm(char *, int, int *);
 
-WINDOW *stdscr;
-int COLORS;
-int COLOR_PAIRS;
-int COLS;
-int LINES;
+extern WINDOW *stdscr;
+extern int COLORS;
+extern int COLOR_PAIRS;
+extern int COLS;
+extern int LINES;

From pypy.commits at gmail.com  Wed Nov 27 02:07:22 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 26 Nov 2019 23:07:22 -0800 (PST)
Subject: [pypy-commit] pypy release-pypy2.7-v7.x: merge default into release
 branch
Message-ID: <5dde20aa.1c69fb81.5e95a.ae9a@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-pypy2.7-v7.x
Changeset: r98161:c649c1ec74b5
Date: 2019-11-27 09:06 +0200
http://bitbucket.org/pypy/pypy/changeset/c649c1ec74b5/

Log:	merge default into release branch

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -99,16 +99,16 @@
   Spenser Bauman
   Michal Bendowski
   Jan de Mooij
+  Stefano Rivera
   Tyler Wade
+  Stefan Beyer
   Vincent Legoll
   Michael Foord
   Stephan Diehl
-  Stefano Rivera
   Jean-Paul Calderone
   Stefan Schwarzer
   Tomek Meka
   Valentino Volonghi
-  Stefan Beyer
   Patrick Maupin
   Devin Jeanpierre
   Bob Ippolito
@@ -137,9 +137,10 @@
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
+  Stian Andreassen
+  Julian Berman
   William Leslie
   Pawe&#322; Piotr Przeradowski
-  Stian Andreassen
   marky1991
   Ilya Osadchiy
   Tobias Oberstein
@@ -150,7 +151,7 @@
   tav
   Georg Brandl
   Joannah Nanjekye
-  Julian Berman
+  Yannick Jadoul
   Bert Freudenberg
   Wanja Saatkamp
   Mike Blume
@@ -275,6 +276,7 @@
   Lutz Paelike
   Ian Foote
   Philipp Rustemeuer
+  Bernd Schoeller
   Logan Chien
   Catalin Gabriel Manciu
   Jacob Oscarson
@@ -302,7 +304,6 @@
   Laurens Van Houtven
   Bobby Impollonia
   Roberto De Ioris
-  Yannick Jadoul
   Jeong YunWon
   Christopher Armstrong
   Aaron Tubbs
@@ -357,6 +358,7 @@
   Daniil Yarancev
   Min RK
   OlivierBlanvillain
+  bernd.schoeller at inf.ethz.ch
   dakarpov at gmail.com
   Jonas Pfannschmidt
   Zearin
@@ -398,6 +400,7 @@
   Jesdi
   Konrad Delong
   Dinu Gherman
+  Sam Edwards
   pizi
   Tom&#225;&#353; Pru&#382;ina
   James Robert
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -65,16 +65,16 @@
   Spenser Bauman
   Michal Bendowski
   Jan de Mooij
+  Stefano Rivera
   Tyler Wade
+  Stefan Beyer
   Vincent Legoll
   Michael Foord
   Stephan Diehl
-  Stefano Rivera
   Jean-Paul Calderone
   Stefan Schwarzer
   Tomek Meka
   Valentino Volonghi
-  Stefan Beyer
   Patrick Maupin
   Devin Jeanpierre
   Bob Ippolito
@@ -103,9 +103,10 @@
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
+  Stian Andreassen
+  Julian Berman
   William Leslie
   Pawe&#322; Piotr Przeradowski
-  Stian Andreassen
   marky1991
   Ilya Osadchiy
   Tobias Oberstein
@@ -116,7 +117,7 @@
   tav
   Georg Brandl
   Joannah Nanjekye
-  Julian Berman
+  Yannick Jadoul
   Bert Freudenberg
   Wanja Saatkamp
   Mike Blume
@@ -241,6 +242,7 @@
   Lutz Paelike
   Ian Foote
   Philipp Rustemeuer
+  Bernd Schoeller
   Logan Chien
   Catalin Gabriel Manciu
   Jacob Oscarson
@@ -253,7 +255,6 @@
   Lene Wagner
   Tomo Cocoa
   Miro Hron&#269;ok
-  Anthony Sottile
   David Lievens
   Neil Blakey-Milner
   Henrik Vendelbo
@@ -269,7 +270,6 @@
   Laurens Van Houtven
   Bobby Impollonia
   Roberto De Ioris
-  Yannick Jadoul
   Jeong YunWon
   Christopher Armstrong
   Aaron Tubbs
@@ -324,6 +324,7 @@
   Daniil Yarancev
   Min RK
   OlivierBlanvillain
+  bernd.schoeller at inf.ethz.ch
   dakarpov at gmail.com
   Jonas Pfannschmidt
   Zearin
@@ -365,6 +366,7 @@
   Jesdi
   Konrad Delong
   Dinu Gherman
+  Sam Edwards
   pizi
   Tom&#225;&#353; Pru&#382;ina
   James Robert
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-v7.3.0.rst
    release-v7.2.0.rst
    release-v7.1.1.rst
    release-v7.1.0.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-7.3.0.rst
    whatsnew-pypy2-7.2.0.rst
    whatsnew-pypy2-7.1.0.rst
    whatsnew-pypy2-7.0.0.rst
@@ -43,6 +44,7 @@
 
 .. toctree::
     whatsnew-pypy3-head.rst
+    whatsnew-pypy3-7.3.0.rst
     whatsnew-pypy3-7.2.0.rst
     whatsnew-pypy3-7.1.0.rst
 
diff --git a/pypy/doc/release-v7.3.0.rst b/pypy/doc/release-v7.3.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-v7.3.0.rst
@@ -0,0 +1,213 @@
+====================================
+PyPy v7.3.0: release of 2.7, and 3.6
+====================================
+
+The PyPy team is proud to release the version 7.3.0 of PyPy, which includes
+two different interpreters:
+
+  - PyPy2.7, which is an interpreter supporting the syntax and the features of
+    Python 2.7 including the stdlib for CPython 2.7.13
+
+  - PyPy3.6: which is an interpreter supporting the syntax and the features of
+    Python 3.6, including the stdlib for CPython 3.6.9.
+    
+The interpreters are based on much the same codebase, thus the double
+release.
+
+We have worked with the python packaging group to support tooling around
+building third party packages for python, so this release changes the ABI tag
+for PyPy.
+
+The `CFFI`_ backend has been updated to version 1.13.1. We recommend using CFFI
+rather than c-extensions to interact with C.
+
+The built-in ``_cppyy`` module was upgraded to 1.10.6, which
+provides, among others, better template resolution, stricter ``enum`` handling,
+anonymous struct/unions, cmake fragments for distribution, optimizations for
+PODs, and faster wrapper calls. We reccomend using cppyy_ for performant
+wrapping of C++ code for Python.
+
+The vendored pyrepl package for interaction inside the REPL was updated.
+
+Support for codepage encoding and decoding was added for Windows.
+
+As always, this release fixed several issues and bugs raised by the growing
+community of PyPy users.  We strongly recommend updating. Many of the fixes are
+the direct result of end-user bug reports, so please continue reporting issues
+as they crop up.
+
+You can download the v7.3 releases here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project. If PyPy is not quite good enough for your needs, we are available for
+direct consulting work.
+
+We would also like to thank our contributors and encourage new people to join
+the project. PyPy has many layers and we need help with all of them: `PyPy`_
+and `RPython`_ documentation improvements, tweaking popular modules to run
+on pypy, or general `help`_ with making RPython's JIT even better. Since the
+previous release, we have accepted contributions from 3 new contributors,
+thanks for pitching in.
+
+.. _`PyPy`: index.html
+.. _`RPython`: https://rpython.readthedocs.org
+.. _`help`: project-ideas.html
+.. _`CFFI`: http://cffi.readthedocs.io
+.. _`cppyy`: https://cppyy.readthedocs.io
+.. _`available as wheels`: https://github.com/antocuni/pypy-wheels
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7, 3.6. It's fast (`PyPy and CPython 2.7.x`_ performance
+comparison) due to its integrated tracing JIT compiler.
+
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
+
+This PyPy release supports:
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
+
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+  * 64-bit **ARM** machines running Linux.
+
+Unfortunately at the moment of writing our ARM buildbots are out of service,
+so for now we are **not** releasing any binary for the ARM architecture (32
+bit), although PyPy does support ARM 32 bit processors. 
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html
+
+
+Changelog
+=========
+
+Changes shared across versions
+------------------------------
+
+* Fix segfault when calling descr-methods with no arguments
+* Change the SOABI and subsequently change the reported ABI tag.
+* Update the builtin ``_cppyy`` backend to 1.10.6
+* Performance improvements in string/unicode handling
+* Fix compilation error when building `revdb`_ (`issue 3084`_, actually
+  released in PyPy 7.2 but not mentioned in the release notes)
+* Add JIT options to documentation and an option for JIT help via ``pypy --jit
+  help``
+* Fix regex escapes in pyrepl (`issue 3097`_)
+* Allow reloading of subclasses of module (`issue 3099`_)
+* Work around a gcc bug, which was reported to them and fixed (`issue 3086`_)
+* Fix (again) faulty logic when decoding invalid UTF-8 (`issue 2389`_)
+* Fix up LICENSE file
+* Turn all ``http`` links in the documentation to ``https``
+* Update the bndled pip and setuptools (used in ``pypy -mensurepip`` to version
+  that support `manylinux2010`_ wheels
+* Link the ``DEFAULT_SOABI`` to the ``PYPY_VERSION``
+* Workaround for programs calling ``sys.setrecursionlimit(huge_value)`` (`issue
+  3094`_)
+* Set minimal ``MACOSX_DEPLOYMENT_TARGET`` to 10.7 on macOS; cpython uses 10.5
+* Fix a JIT bug causing rare register corruption on aarch64
+* Add discovery of ``ncursesw`` when building ``_minimal_curses`` and improve
+  handling of old curses versions (`issue 2970`_)
+* Improve the error message for ``class X(random_object)`` (`issue 3109`_)
+* Deal with json dicts containing repeated keys in the new map based parser
+  (`issue 3108`_)
+* Port parts of the `portable pypy`_ repackaging scripts to add an option for
+  ``RPATH`` manipulation on linux
+* Check for overflow in ctypes array creation
+* Better support and report MSVC versions used to compile on windows
+* Allow any kind of buffer in socket.setsockopt(), like CPython (`issue 3114`_)
+
+C-API (cpyext) and c-extensions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* Add ``_PySet_Next``, ``_PySet_NextEntry``
+* Correctly swallow exceptions happening inside ``PyDict_GetItem()`` (`issue
+  3098`_)
+* Respect tp_dict on PyType_Ready (`issue XXX`_)
+* Allow calling ``PyType_Ready`` on a subclass with a partially built
+  ``tp_base`` (issue 3117`_)
+* Rename ``tuple_new`` to ``_PyPy_tuple_new`` to follow the naming convention of
+  exported symbols in ``libpypy-c.so``
+* Actually restore the traceback in ``PyErr_Restore`` (`issue 3120`_)
+
+Python 3.6 only
+---------------
+
+* Don't grow the ``lzma.decompress()`` buffer past ``max_length`` (`issue 3088`_)
+* Backport fix from CPython for failure of ``lzma`` to decompress a file
+  (`issue 3090`_)
+* Fix ``asyncgen_hooks`` and refactor ``coroutine execution``
+* Fix range checking in GB18030 decoder (CPython issue `29990`_)
+* Fix handling escape characters in HZ codec (CPython issue `30003`_)
+* Reject null characters in a few more functions (CPython issue `13617`_)
+* Fix build on macOS without ``clock_gettime`` (before 10.12 and xcode 8,
+  released 2016)
+* Backport 3.7.5 changes to ``timedelta.__eq__`` and ``time.__eq__`` (CPython
+  issue `37579`_)
+* Backport more fixes to comparisons in ``datetime.py`` (CPython issue `37985`_)
+* Use the python tag in ``pyc`` file names, not the abi tag
+* Handle string formatting with a single ``[`` in the format string (`issue
+  3100`_)
+* Backport some of the patches in `macports pypy`_
+* Add missing ``HAVE_FACCESSAT`` to ``posix._have_functions``
+* Update pyrepl from upstream package (`issue 2971`_)
+* Fix ``PyFrame._guess_function_name_parens()``
+* Fix range of allowed years in ``time.mktime`` to match CPython `13312`_
+* Generators need to store the old current ``exc_info`` in a place that is
+  visible, because in one corner case a call to ``sys.exc_info()`` might need
+  it.  (`issue 3096`_)
+* Remove incorrect clobbering of the ``locals`` after running ``exec()``
+* Adds encoding, decoding codepages on win32
+
+Python 3.6 C-API
+~~~~~~~~~~~~~~~~
+
+* Add ``PyObject_GenericGetDict``, ``PyObject_GenericSetDict``, ``_Py_strhex``,
+  ``_Py_strhex_bytes``, ``PyUnicodeNew``, ``_PyFinalizing``,
+  ``PySlice_Unpack``, ``PySlice_AdjustIndices``
+* Implement ``pystrhex.h`` (`issue 2687`_)
+* Make ``PyUnicodeObject`` slightly more compact
+* Fix memory leak when releasing a ``PyUnicodeObject``
+
+.. _`revdb`: fix broken link
+.. _`portable pypy`: fix broken link
+.. _`manylinux2010`: fix broken link
+.. _`macports pypy`: https://github.com/macports/macports-ports/blob/master/lang/pypy/files/darwin.py.diff
+
+.. _`issue 2389`: https://bitbucket.com/pypy/pypy/issues/2389
+.. _`issue 2687`: https://bitbucket.com/pypy/pypy/issues/2687
+.. _`issue 2970`: https://bitbucket.com/pypy/pypy/issues/2970
+.. _`issue 2971`: https://bitbucket.com/pypy/pypy/issues/2971
+.. _`issue 3084`: https://bitbucket.com/pypy/pypy/issues/3084
+.. _`issue 3086`: https://bitbucket.com/pypy/pypy/issues/3086
+.. _`issue 3088`: https://bitbucket.com/pypy/pypy/issues/3088
+.. _`issue 3090`: https://bitbucket.com/pypy/pypy/issues/3090
+.. _`issue 3094`: https://bitbucket.com/pypy/pypy/issues/3094
+.. _`issue 3096`: https://bitbucket.com/pypy/pypy/issues/3096
+.. _`issue 3097`: https://bitbucket.com/pypy/pypy/issues/3097
+.. _`issue 3098`: https://bitbucket.com/pypy/pypy/issues/3098
+.. _`issue 3099`: https://bitbucket.com/pypy/pypy/issues/3099
+.. _`issue 3100`: https://bitbucket.com/pypy/pypy/issues/3100
+.. _`issue 3108`: https://bitbucket.com/pypy/pypy/issues/3108
+.. _`issue 3109`: https://bitbucket.com/pypy/pypy/issues/3109
+.. _`issue 3114`: https://bitbucket.com/pypy/pypy/issues/3114
+.. _`issue 3117`: https://bitbucket.com/pypy/pypy/issues/3117
+.. _`issue 3120`: https://bitbucket.com/pypy/pypy/issues/3120
+
+.. _13312: https://bugs.python.org/issue13312
+.. _13617: https://bugs.python.org/issue13617
+.. _29990: https://bugs.python.org/issue29990
+.. _30003: https://bugs.python.org/issue30003
+.. _37579: https://bugs.python.org/issue37579
+.. _37985: https://bugs.python.org/issue37985
+.. _37985: https://bugs.python.org/issue37985
+
+
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -1,33 +1,8 @@
-==========================
-What's new in PyPy2.7 7.3+
-==========================
+============================
+What's new in PyPy2.7 7.3.0+
+============================
 
-.. this is a revision shortly after release-pypy-7.2.0
-.. startrev: a511d86377d6 
+.. this is a revision shortly after release-pypy-7.3.0
+.. startrev: dbbbae99135f 
 
-.. branch: fix-descrmismatch-crash
 
-Fix segfault when calling descr-methods with no arguments
-
-.. branch: https-readme
-
-Convert http -> https in README.rst
-
-.. branch: license-update
-
-Update list directories in LICENSE
-
-.. branch: allow-forcing-no-embed
-
-When packaging, allow suppressing embedded dependencies via
-PYPY_NO_EMBED_DEPENDENCIES
-
-.. branch: int-test-is-zero
-
-.. branch: cppyy-dev
-
-Upgraded the built-in ``_cppyy`` module to ``cppyy-backend 1.10.6``, which
-provides, among others, better template resolution, stricter ``enum`` handling,
-anonymous struct/unions, cmake fragments for distribution, optimizations for
-PODs, and faster wrapper calls.
-
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-pypy2-7.3.0.rst
copy from pypy/doc/whatsnew-head.rst
copy to pypy/doc/whatsnew-pypy2-7.3.0.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-pypy2-7.3.0.rst
@@ -1,6 +1,6 @@
-==========================
-What's new in PyPy2.7 7.3+
-==========================
+===========================
+What's new in PyPy2.7 7.3.0
+===========================
 
 .. this is a revision shortly after release-pypy-7.2.0
 .. startrev: a511d86377d6 
diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-7.3.0.rst
copy from pypy/doc/whatsnew-pypy3-head.rst
copy to pypy/doc/whatsnew-pypy3-7.3.0.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-7.3.0.rst
@@ -1,6 +1,6 @@
-========================
-What's new in PyPy3 7.2+
-========================
+=========================
+What's new in PyPy3 7.3.0
+=========================
 
 .. this is the revision after release-pypy3.6-v7.2
 .. startrev: 6d2f8470165b
diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -1,7 +1,7 @@
-========================
-What's new in PyPy3 7.2+
-========================
+==========================
+What's new in PyPy3 7.3.0+
+==========================
 
-.. this is the revision after release-pypy3.6-v7.2
-.. startrev: 6d2f8470165b
+.. this is the revision after release-pypy3.6-v7.3.0
+.. startrev: 78b4d0a7cf2e
 

From pypy.commits at gmail.com  Wed Nov 27 08:01:30 2019
From: pypy.commits at gmail.com (stevie_92)
Date: Wed, 27 Nov 2019 05:01:30 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-gc-cycle: Fixed bug in rrc incmark
Message-ID: <5dde73aa.1c69fb81.2366d.8a84@mx.google.com>

Author: Stefan Beyer <home at sbeyer.at>
Branch: cpyext-gc-cycle
Changeset: r98162:93669df5242d
Date: 2019-11-27 14:00 +0100
http://bitbucket.org/pypy/pypy/changeset/93669df5242d/

Log:	Fixed bug in rrc incmark

diff --git a/rpython/memory/gc/rrc/incmark.py b/rpython/memory/gc/rrc/incmark.py
--- a/rpython/memory/gc/rrc/incmark.py
+++ b/rpython/memory/gc/rrc/incmark.py
@@ -238,8 +238,8 @@
             while pygchdr <> self.pyobj_list:
                 self._pyobj_gc_refcnt_set(pygchdr, 1)
                 pygchdr = pygchdr.c_gc_next
+            # resurrect "dead" objects
             pygchdr = self.pyobj_old_list.c_gc_next
-            # resurrect "dead" objects
             while pygchdr <> self.pyobj_old_list:
                 self._pyobj_gc_refcnt_set(pygchdr, 1)
                 pygchdr = pygchdr.c_gc_next
@@ -254,6 +254,7 @@
         start = time.time()
 
         if isolate_consistent:
+            debug_print("isolate consistent")
             if not self._gc_list_is_empty(self.pyobj_isolate_old_list):
                 self._gc_list_merge(self.pyobj_isolate_old_list,
                                     self.pyobj_list)
@@ -261,12 +262,14 @@
                 self._gc_list_merge(self.pyobj_isolate_dead_list,
                                     self.pyobj_dead_list)
         else:
+            debug_print("isolate inconsistent")
             # continue previous loop, keep objects alive
             pygchdr = pygchdr_continue_isolate
             while pygchdr <> self.pyobj_isolate_old_list:
                 self._pyobj_gc_refcnt_set(pygchdr, 1)
                 pygchdr = pygchdr.c_gc_next
             # resurrect "dead" objects
+            pygchdr = self.pyobj_isolate_dead_list.c_gc_next
             while pygchdr <> self.pyobj_isolate_dead_list:
                 self._pyobj_gc_refcnt_set(pygchdr, 1)
                 pygchdr = pygchdr.c_gc_next

From pypy.commits at gmail.com  Wed Nov 27 19:18:36 2019
From: pypy.commits at gmail.com (antocuni)
Date: Wed, 27 Nov 2019 16:18:36 -0800 (PST)
Subject: [pypy-commit] pypy hpy-ctypespace: close branch which will be
 merged into hpy
Message-ID: <5ddf125c.1c69fb81.e9e54.b51d@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy-ctypespace
Changeset: r98163:ed73b0a1ed73
Date: 2019-11-28 01:11 +0100
http://bitbucket.org/pypy/pypy/changeset/ed73b0a1ed73/

Log:	close branch which will be merged into hpy


From pypy.commits at gmail.com  Wed Nov 27 19:18:39 2019
From: pypy.commits at gmail.com (antocuni)
Date: Wed, 27 Nov 2019 16:18:39 -0800 (PST)
Subject: [pypy-commit] pypy hpy: Merge branch hpy-ctypespace. Use CTypeSpace
 to declare HPy types
Message-ID: <5ddf125f.1c69fb81.f348.aa0f@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98164:3239497d9665
Date: 2019-11-28 01:17 +0100
http://bitbucket.org/pypy/pypy/changeset/3239497d9665/

Log:	Merge branch hpy-ctypespace. Use CTypeSpace to declare HPy types

	I think that the final result is much easier to read and to manage.
	Currently, the C bits needs to be written by hand, but we can
	probably hack more until it can read the real universal/hpy.h or an
	autogen version which is easier to parse.

diff --git a/pypy/module/hpy_universal/_vendored/include/universal/hpy.h b/pypy/module/hpy_universal/_vendored/include/universal/hpy.h
--- a/pypy/module/hpy_universal/_vendored/include/universal/hpy.h
+++ b/pypy/module/hpy_universal/_vendored/include/universal/hpy.h
@@ -6,7 +6,11 @@
 #include <stdarg.h>
 
 typedef intptr_t HPy_ssize_t;
-typedef struct { HPy_ssize_t _i; } HPy;
+
+// WARNING: the following change has been done inside the pypy hpy-ctypespace
+// branch. If/when the branch is merged, we should backport it to pyhandle/hpy
+struct _HPy_s { HPy_ssize_t _i; };
+typedef struct _HPy_s HPy;
 
 typedef struct _HPyContext_s *HPyContext;
 struct _object;  /* that's PyObject inside CPython */
diff --git a/pypy/module/hpy_universal/interp_extfunc.py b/pypy/module/hpy_universal/interp_extfunc.py
--- a/pypy/module/hpy_universal/interp_extfunc.py
+++ b/pypy/module/hpy_universal/interp_extfunc.py
@@ -14,15 +14,14 @@
     def __init__(self, ml, w_self):
         self.ml = ml
         self.w_self = w_self
-        self.name = rffi.charp2str(self.ml.c_ml_name)
+        self.name = rffi.constcharp2str(self.ml.c_ml_name)
         self.flags = rffi.cast(lltype.Signed, self.ml.c_ml_flags)
         # fetch the real HPy function pointer, by calling ml_meth, which
         # is a function that returns it and also the CPython-only trampoline
         with lltype.scoped_alloc(
-                rffi.CArray(llapi._HPyCFunctionPtr), 1) as funcptr:
+                rffi.CArray(llapi._HPyCFunction), 1) as funcptr:
             with lltype.scoped_alloc(
-                    rffi.CArray(llapi._HPy_CPyCFunctionPtr), 1)  \
-                                                        as ignored_trampoline:
+                    rffi.CArray(llapi._HPyCPyCFunction), 1) as ignored_trampoline:
                 ml.c_ml_meth(funcptr, ignored_trampoline)
                 self.cfuncptr = funcptr[0]
 
diff --git a/pypy/module/hpy_universal/interp_hpy.py b/pypy/module/hpy_universal/interp_hpy.py
--- a/pypy/module/hpy_universal/interp_hpy.py
+++ b/pypy/module/hpy_universal/interp_hpy.py
@@ -32,7 +32,7 @@
 
 @apifunc([llapi.HPyContext, lltype.Ptr(llapi.HPyModuleDef)], llapi.HPy)
 def HPyModule_Create(space, ctx, hpydef):
-    modname = rffi.charp2str(hpydef.c_m_name)
+    modname = rffi.constcharp2str(hpydef.c_m_name)
     w_mod = Module(space, space.newtext(modname))
     #
     # add all the functions defined in hpydef.c_m_methods
@@ -100,7 +100,7 @@
 
 def create_hpy_module(space, name, origin, lib, initfunc):
     state = space.fromcache(State)
-    initfunc = rffi.cast(llapi.HPyInitFuncPtr, initfunc)
+    initfunc = rffi.cast(llapi.HPyInitFunc, initfunc)
     h_module = generic_cpy_call_dont_convert_result(space, initfunc, state.ctx)
     return handles.consume(space, h_module)
 
diff --git a/pypy/module/hpy_universal/llapi.py b/pypy/module/hpy_universal/llapi.py
--- a/pypy/module/hpy_universal/llapi.py
+++ b/pypy/module/hpy_universal/llapi.py
@@ -3,12 +3,13 @@
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.translator import cdir
 from pypy import pypydir
-from pypy.module.hpy_universal import _vendored
+from pypy.module.cpyext.cparser import CTypeSpace
 
 PYPYDIR = py.path.local(pypydir)
 INCLUDE_DIR = PYPYDIR.join('module', 'hpy_universal', '_vendored', 'include')
 SRC_DIR = PYPYDIR.join('module', 'hpy_universal', 'src')
 
+
 eci = ExternalCompilationInfo(
     includes=["universal/hpy.h", "getargs.h"],
     include_dirs=[
@@ -21,6 +22,11 @@
     ],
     post_include_bits=["""
         RPY_EXTERN void *_HPy_GetGlobalCtx(void);
+
+        // these are workarounds for a CTypeSpace limitation, since it can't properly
+        // handle struct types which are not typedefs
+        typedef struct _HPyContext_s _struct_HPyContext_s;
+        typedef struct _HPy_s _struct_HPy_s;
     """],
     separate_module_sources=["""
         struct _HPyContext_s hpy_global_ctx;
@@ -30,71 +36,89 @@
         }
     """])
 
+cts = CTypeSpace()
+# NOTE: the following C source is NOT seen by the C compiler during
+# translation: it is used only as a nice way to declare the lltype.* types
+# which are needed here
+cts.headers.append('stdint.h')
+cts.parse_source("""
+typedef intptr_t HPy_ssize_t;
 
-HPy_ssize_t = lltype.Signed # XXXXXXXXX?
+// see below for more info about HPy vs _struct_HPy_s
+typedef struct _HPy_s {
+    HPy_ssize_t _i;
+} _struct_HPy_s;
+typedef HPy_ssize_t HPy;
+
+typedef struct _HPyContext_s {
+    int ctx_version;
+    struct _HPy_s h_None;
+    struct _HPy_s h_True;
+    struct _HPy_s h_False;
+    struct _HPy_s h_ValueError;
+    void *ctx_Module_Create;
+    void *ctx_Dup;
+    void *ctx_Close;
+    void *ctx_Long_FromLong;
+    void *ctx_Long_AsLong;
+    void *ctx_Arg_Parse;
+    void *ctx_Number_Add;
+    void *ctx_Unicode_FromString;
+    void *ctx_Err_SetString;
+    void *ctx_FromPyObject;
+    void *ctx_AsPyObject;
+    void *ctx_CallRealFunctionFromTrampoline;
+} _struct_HPyContext_s;
+typedef struct _HPyContext_s *HPyContext;
+
+typedef HPy (*HPyInitFunc)(HPyContext ctx);
+
+typedef HPy (*_HPyCFunction)(HPyContext ctx, HPy self, HPy args);
+typedef void *_HPyCPyCFunction; // not used here
+typedef void (*_HPyMethodPairFunc)(_HPyCFunction *out_func,
+                                   _HPyCPyCFunction *out_trampoline);
+typedef HPy (*HPyMeth_VarArgs)(HPyContext ctx, HPy self, HPy *args, HPy_ssize_t nargs);
+
+typedef struct {
+    const char         *ml_name;
+    _HPyMethodPairFunc ml_meth;
+    int                ml_flags;
+    const char         *ml_doc;
+} HPyMethodDef;
+
+typedef struct {
+    void *dummy; // this is needed because we put a comma after HPyModuleDef_HEAD_INIT :(
+    const char* m_name;
+    const char* m_doc;
+    HPy_ssize_t m_size;
+    HPyMethodDef *m_methods;
+} HPyModuleDef;
+""")
+
+HPy_ssize_t = cts.gettype('HPy_ssize_t')
+# XXX: HPyContext is equivalent to the old HPyContext which was defined
+# explicitly using rffi.CStruct: the only different is that this is missing
+# hints={'eci': eci}: however, the tests still pass (including
+# ztranslation). Why was the eci needed?
+HPyContext = cts.gettype('HPyContext')
 
 # for practical reason, we use a primitive type to represent HPy almost
-# everywhere in RPython. HOWEVER, the "real" HPy C type which is defined in
-# universal/hpy.h is an anonymous struct: we need to use it e.g. to represent
-# fields inside HPyContextS
-HPy = HPy_ssize_t
-HPyS_real = rffi.CStruct('HPy',
-    ('_i', HPy_ssize_t),
-    hints={'eci': eci, 'typedef': True},
-)
+# everywhere in RPython: for example, rffi cannot handle functions returning
+# structs. HOWEVER, the "real" HPy C type is a struct, which is available as
+# "_struct_HPy_s"
+HPy = cts.gettype('HPy')
 
+HPyInitFunc = cts.gettype('HPyInitFunc')
+_HPyCFunction = cts.gettype('_HPyCFunction')
+_HPyCPyCFunction = cts.gettype('_HPyCPyCFunction')
+HPyMeth_VarArgs = cts.gettype('HPyMeth_VarArgs')
 
-HPyContextS = rffi.CStruct('_HPyContext_s',
-    ('ctx_version', rffi.INT_real),
-    ('h_None', HPyS_real),
-    ('h_True', HPyS_real),
-    ('h_False', HPyS_real),
-    ('h_ValueError', HPyS_real),
-    ('ctx_Module_Create', rffi.VOIDP),
-    ('ctx_Dup', rffi.VOIDP),
-    ('ctx_Close', rffi.VOIDP),
-    ('ctx_Long_FromLong', rffi.VOIDP),
-    ('ctx_Long_AsLong', rffi.VOIDP),
-    ('ctx_Arg_Parse', rffi.VOIDP),
-    ('ctx_Number_Add', rffi.VOIDP),
-    ('ctx_Unicode_FromString', rffi.VOIDP),
-    ('ctx_Err_SetString', rffi.VOIDP),
-    ('ctx_FromPyObject', rffi.VOIDP),
-    ('ctx_AsPyObject', rffi.VOIDP),
-    ('ctx_CallRealFunctionFromTrampoline', rffi.VOIDP),
-    hints={'eci': eci},
-)
-HPyContext = lltype.Ptr(HPyContextS)
-HPyInitFuncPtr = lltype.Ptr(lltype.FuncType([HPyContext], HPy))
-
-_HPyCFunctionPtr = lltype.Ptr(lltype.FuncType([HPyContext, HPy, HPy], HPy))
-_HPy_CPyCFunctionPtr = rffi.VOIDP    # not used here
-
-HPyMeth_VarArgs = lltype.Ptr(
-    lltype.FuncType([HPyContext, HPy, lltype.Ptr(rffi.CArray(HPy)), HPy_ssize_t], HPy))
-
-
-_HPyMethodPairFuncPtr = lltype.Ptr(lltype.FuncType([
-        rffi.CArrayPtr(_HPyCFunctionPtr),
-        rffi.CArrayPtr(_HPy_CPyCFunctionPtr)],
-    lltype.Void))
-
-HPyMethodDef = rffi.CStruct('HPyMethodDef',
-    ('ml_name', rffi.CCHARP),
-    ('ml_meth', _HPyMethodPairFuncPtr),
-    ('ml_flags', rffi.INT_real),
-    ('ml_doc', rffi.CCHARP),
-    hints={'eci': eci, 'typedef': True},
-)
-
-HPyModuleDef = rffi.CStruct('HPyModuleDef',
-    ('dummy', rffi.VOIDP),
-    ('m_name', rffi.CCHARP),
-    ('m_doc', rffi.CCHARP),
-    ('m_size', lltype.Signed),
-    ('m_methods', rffi.CArrayPtr(HPyMethodDef)),
-    hints={'eci': eci, 'typedef': True},
-)
+HPyMethodDef = cts.gettype('HPyMethodDef')
+HPyModuleDef = cts.gettype('HPyModuleDef')
+# CTypeSpace converts "HPyMethodDef*" into lltype.Ptr(HPyMethodDef), but we
+# want a CArrayPtr instead, so that we can index the items inside
+# HPyModule_Create
+HPyModuleDef._flds['c_m_methods'] = rffi.CArrayPtr(HPyMethodDef)
 
 METH_VARARGS  = 0x0001
 METH_KEYWORDS = 0x0002
diff --git a/rpython/jit/backend/llsupport/test/test_descr.py b/rpython/jit/backend/llsupport/test/test_descr.py
--- a/rpython/jit/backend/llsupport/test/test_descr.py
+++ b/rpython/jit/backend/llsupport/test/test_descr.py
@@ -426,9 +426,10 @@
     #
     fielddescr = get_field_arraylen_descr(c0, rstr.STR)
     ofs = fielddescr.offset
-    assert repr(ofs) == ("< <FieldOffset <GcStruct rpy_string { hash, chars }>"
-                         " 'chars'> + < ArrayLengthOffset"
-                         " <Array of Char > > >")
+    assert repr(ofs) == (
+        "< <FieldOffset <GcStruct rpy_string { hash, chars }> 'chars'> "
+        "+ < ArrayLengthOffset <Array of Char "
+        "{'extra_item_after_alloc': 1, 'immutable': True} > > >")
     # caching:
     assert fielddescr is get_field_arraylen_descr(c0, rstr.STR)
 
diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py
--- a/rpython/rtyper/lltypesystem/lltype.py
+++ b/rpython/rtyper/lltypesystem/lltype.py
@@ -467,12 +467,16 @@
     _str_fields = saferecursive(_str_fields, '...')
 
     def __str__(self):
-        return "%s of %s " % (self.__class__.__name__,
-                               self._str_fields(),)
+        hints = (' ' + str(self._hints)) if self._hints else ''
+        return "%s of %s%s " % (self.__class__.__name__,
+                                self._str_fields(),
+                                hints)
 
     def _short_name(self):
-        return "%s %s" % (self.__class__.__name__,
-                          self.OF._short_name(),)
+        hints = (' ' + str(self._hints)) if self._hints else ''
+        return "%s %s%s" % (self.__class__.__name__,
+                            self.OF._short_name(),
+                            hints)
     _short_name = saferecursive(_short_name, '...')
 
     def _container_example(self):
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -1024,6 +1024,14 @@
  ) = make_string_mappings(unicode)
 
 
+def constcharp2str(cp):
+    """
+    Like charp2str, but takes a CONST_CCHARP instead
+    """
+    cp = cast(CCHARP, cp)
+    return charp2str(cp)
+constcharp2str._annenforceargs_ = [lltype.SomePtr(CONST_CCHARP)]
+
 @not_rpython
 def _deprecated_get_nonmovingbuffer(*args):
     raise Exception(
diff --git a/rpython/rtyper/lltypesystem/test/test_rffi.py b/rpython/rtyper/lltypesystem/test/test_rffi.py
--- a/rpython/rtyper/lltypesystem/test/test_rffi.py
+++ b/rpython/rtyper/lltypesystem/test/test_rffi.py
@@ -142,6 +142,33 @@
         xf = self.compile(f, [], backendopt=False)
         assert xf() == 3
 
+    def test_constcharp2str(self):
+        c_source = py.code.Source("""
+        const char *z(void)
+        {
+            return "hello world";
+        }
+        """)
+        eci = ExternalCompilationInfo(separate_module_sources=[c_source],
+                                     post_include_bits=['const char *z(void);'])
+        z = llexternal('z', [], CONST_CCHARP, compilation_info=eci)
+
+        def f():
+            l_buf = lltype.malloc(CCHARP.TO, 5, flavor='raw')
+            l_buf[0] = 'A'
+            l_buf[1] = 'B'
+            l_buf[2] = 'C'
+            l_buf[3] = '\x00'
+            l_buf[4] = 'E'
+            l_constbuf = cast(CONST_CCHARP, l_buf)
+            res = constcharp2str(l_constbuf)
+            lltype.free(l_buf, flavor='raw')
+            return len(res)
+
+        assert f() == 3
+        xf = self.compile(f, [], backendopt=False)
+        assert xf() == 3
+
     def test_stringstar(self):
         c_source = """
         #include <string.h>
@@ -813,7 +840,7 @@
     interpret(test_ptradd, [])
 
 def test_voidptr():
-    assert repr(VOIDP) == "<* Array of void >"
+    assert repr(VOIDP) == "<* Array of void {'nolength': True, 'render_as_void': True} >"
 
 class TestCRffi(BaseTestRffi):
     def compile(self, func, args, **kwds):

From pypy.commits at gmail.com  Thu Nov 28 03:37:14 2019
From: pypy.commits at gmail.com (Matti Picus)
Date: Thu, 28 Nov 2019 00:37:14 -0800 (PST)
Subject: [pypy-commit] buildbot default: tweaks to get started,
 add 32 bit dockerfile
Message-ID: <5ddf873a.1c69fb81.96c74.7384@mx.google.com>

Author: Matti Picus
Branch: 
Changeset: r1107:39d00880d7ef
Date: 2019-11-28 09:36 +0100
http://bitbucket.org/pypy/buildbot/changeset/39d00880d7ef/

Log:	tweaks to get started, add 32 bit dockerfile

diff --git a/docker/Dockerfile b/docker/Dockerfile
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -5,7 +5,7 @@
 # docker build -t buildslave -f docker/Dockerfile random_name 
 #
 # To create the buildslave configuration, call
-# docker run --rm --user $UID -v<abspath/to/builder/dir>:/build_dir> \
+# docker run --rm --user buildslave -v<abspath/to/builder/dir>:/build_dir> \
 #    -eSLAVENAME=<name> -ePASSWORD=<password> buildslave
 # Then you can examine the <build_dir>/buildbot.tac file.
 #
@@ -20,54 +20,56 @@
 # Use PYPY_MAKE_PORTABLE to package a built PyPy in a portable fashion (that
 # is the motivation for doing this in the first place, right?)
 #
-# docker run --rm --user $UID -v<abspath/to/builder/dir>:/build_dir> \
+# docker run --rm --user buildslave -v<abspath/to/builder/dir>:/build_dir> \
 #     -ePYPY_MAKE_PORTABLE=1 buildslave
 #
+# Use HOME since the default for docker is /, which
+# is not avaialbe to the buildslave user
+#
 # You might want to keep the PYPY_USESSION_DIR where the testing/building
 # artifacts are. Docker will not do this for you, so do something like this
 # to save the files outside the docker
 #
 # mkdir -p build_dir/tmp
-# docker run --rm --user $UID -v<abspath/to/builder/dir>:/build_dir> \
-#     -ePYPY_USESSION_DIR=/build_dir/tmp -ePYPY_MAKE_PORTABLE=1 buildslave
+# docker run --rm --user buildslave -v<abspath/to/builder/dir>:/build_dir> \
+#     -ePYPY_USESSION_DIR=/build_dir/tmp -ePYPY_MAKE_PORTABLE=1 \
+#     -eHOME=/buildslave_home -v/home/buildslave:/buildslave_home buildslave
 #
 # To enter the buildslave image, add a shell command to the end
 #
-# docker run -it --user $UID -v<abspath/to/builder/dir>:/build_dir> \
+# docker run -it --user buildslave -v<abspath/to/builder/dir>:/build_dir> \
 #     -ePYPY_USESSION_DIR=/build_dir/tmp buildslave /bin/bash
 #
 
-#FROM quay.io/pypa/manylinux2010_i686:latest
 FROM quay.io/pypa/manylinux2010_x86_64:latest
 WORKDIR /root
 
 RUN yum -y update
 RUN yum install -y wget bzip2-devel zlib-devel glibc-devel libX11-devel \
    libXt-devel patch expat-devel libXft-devel tk-devel gdbm-devel \
-   perl xz-devel ncurses-devel sqlite-devel prelink
-
-# get a pypy for translation
-RUN wget -q https://bitbucket.org/squeaky/portable-pypy/downloads/pypy-7.0.0-linux_x86_64-portable.tar.bz2 -O - | tar -C /opt -xj
-RUN ln -s /opt/pypy-7.0.0-linux_x86_64-portable/bin/pypy /usr/local/bin/pypy
+   perl xz-devel ncurses-devel sqlite-devel prelink python-virtualenv pypy
 
 # Taken from pyca/infra/cryptography-manylinux
 # centos6 libffi is buggy, download and use a newer one
 # also use the version of openssl that latests pyca/cryptography uses
+ENV PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
+ADD install_openssl.sh /root/install_openssl.sh
 ADD install_libffi.sh /root/install_libffi.sh
-ADD install_openssl.sh /root/install_openssl.sh
 RUN sh install_libffi.sh manylinux2010 2>&1 | tee /root/install_libffi.log
 RUN sh install_openssl.sh manylinux2010 2>&1 | tee /root/install_openssl.log
 
-RUN yum install -y python-virtualenv
+# get a pypy for translation. On x86_64 we can use portable pypy
+RUN wget -q https://bitbucket.org/squeaky/portable-pypy/downloads/pypy-7.0.0-linux_x86_64-portable.tar.bz2 -O - | tar -C /opt -xj
+RUN ln -s /opt/pypy-7.0.0-linux_x86_64-portable/bin/pypy /usr/local/bin/pypy
+
 RUN yum clean packages
 
-# build a python 2.7 virtualenv, use the /opt/python/cp27-cp27m/bin/python
+# build a python 2.7 virtualenv, use the /opt/python/cp27-cp27mu/bin/python
 # since centos6 itself has python2.6
-RUN virtualenv -p /opt/python/cp27-cp27m/bin/python /python27_virt
+RUN virtualenv -p /opt/python/cp27-cp27mu/bin/python /python27_virt
 ENV PATH=/python27_virt/bin:$PATH
 RUN pip install --upgrade pip setuptools
-RUN pip install buildbot-slave pytest hypothesis cffi vmprof mercurial
-ENV PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
+RUN pip install buildbot-slave pytest hypothesis cffi vmprof mercurial virtualenv
 
 CMD if [ -e /build_dir/buildbot.tac ]; then \
     buildslave start --nodaemon /build_dir; \
diff --git a/docker/Dockerfile32 b/docker/Dockerfile32
new file mode 100644
--- /dev/null
+++ b/docker/Dockerfile32
@@ -0,0 +1,79 @@
+# Build with something like this, where
+#  -t is the name of the image
+#  -f is this file
+#  the random_name is a random directory
+# linux32 docker build -t buildslave -f docker/Dockerfile random_name 
+#
+# To create the buildslave configuration, call
+# docker run --rm --user buildslave -v<abspath/to/builder/dir>:/build_dir> \
+#    -eSLAVENAME=<name> -ePASSWORD=<password> buildslave
+# Then you can examine the <build_dir>/buildbot.tac file.
+#
+# To run the buildslave (after the stage above succeeds) you no longer need the
+# SLAVENAME and PASSWORD. The slave will run non-deamonized, which will appear
+# to "hang" the console running the slave. To stop the slave, simply CTRL-C or
+# kill the process.
+#
+# Don't forget to `docker container prune` once in a while to reclaim disk
+# space from stopped containers.
+#
+# Use PYPY_MAKE_PORTABLE to package a built PyPy in a portable fashion (that
+# is the motivation for doing this in the first place, right?)
+#
+# linux32 docker run --rm --user buildslave -v<abspath/to/builder/dir>:/build_dir> \
+#     -ePYPY_MAKE_PORTABLE=1 buildslave
+#
+# Use HOME since the default for docker is /, which
+# is not avaialbe to the buildslave user
+#
+# You might want to keep the PYPY_USESSION_DIR where the testing/building
+# artifacts are. Docker will not do this for you, so do something like this
+# to save the files outside the docker
+#
+# mkdir -p build_dir/tmp
+# linux32 docker run --rm --user buildslave -v<abspath/to/builder/dir>:/build_dir> \
+#     -ePYPY_USESSION_DIR=/build_dir/tmp -ePYPY_MAKE_PORTABLE=1 \
+#     -eHOME=/buildslave_home -v/home/buildslave:/buildslave_home buildslave
+#
+# To enter the buildslave image, add a shell command to the end
+#
+# linux32 docker run -it --user buildslave -v<abspath/to/builder/dir>:/build_dir> \
+#     -ePYPY_USESSION_DIR=/build_dir/tmp buildslave /bin/bash
+#
+
+FROM quay.io/pypa/manylinux2010_i686:latest
+WORKDIR /root
+
+RUN yum -y update
+RUN yum install -y wget bzip2-devel zlib-devel glibc-devel libX11-devel \
+   libXt-devel patch expat-devel libXft-devel tk-devel gdbm-devel \
+   perl xz-devel ncurses-devel sqlite-devel prelink python-virtualenv pypy
+
+# Taken from pyca/infra/cryptography-manylinux
+# centos6 libffi is buggy, download and use a newer one
+# also use the version of openssl that latests pyca/cryptography uses
+ENV PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
+ADD install_openssl.sh /root/install_openssl.sh
+ADD install_libffi.sh /root/install_libffi.sh
+RUN sh install_libffi.sh manylinux2010 m32 2>&1 | tee /root/install_libffi.log
+RUN sh install_openssl.sh manylinux2010 m32 2>&1 | tee /root/install_openssl.log
+
+# get a pypy for translation. On x86_64 we can use portable pypy
+# on i686 we use the old one provided by yum
+# RUN wget -q https://bitbucket.org/squeaky/portable-pypy/downloads/pypy-7.0.0-linux_x86_64-portable.tar.bz2 -O - | tar -C /opt -xj
+# RUN ln -s /opt/pypy-7.0.0-linux_x86_64-portable/bin/pypy /usr/local/bin/pypy
+
+RUN yum clean packages
+
+# build a python 2.7 virtualenv, use the /opt/python/cp27-cp27mu/bin/python
+# since centos6 itself has python2.6
+RUN virtualenv -p /opt/python/cp27-cp27mu/bin/python /python27_virt
+ENV PATH=/python27_virt/bin:$PATH
+RUN pip install --upgrade pip setuptools
+RUN pip install buildbot-slave pytest hypothesis cffi vmprof mercurial virtualenv
+
+CMD if [ -e /build_dir/buildbot.tac ]; then \
+    buildslave start --nodaemon /build_dir; \
+  else \
+    buildslave create-slave /build_dir buildbot.pypy.org:10407 $SLAVENAME $PASSWORD; \
+  fi
diff --git a/docker/install_libffi.sh b/docker/install_libffi.sh
--- a/docker/install_libffi.sh
+++ b/docker/install_libffi.sh
@@ -12,7 +12,7 @@
     rm "${fname}.sha256"
 }
 
-curl -#O "https://mirrors.ocf.berkeley.edu/debian/pool/main/libf/libffi/libffi_${LIBFFI_VERSION}.orig.tar.gz"
+curl -q -#O "https://mirrors.ocf.berkeley.edu/debian/pool/main/libf/libffi/libffi_${LIBFFI_VERSION}.orig.tar.gz"
 check_sha256sum "libffi_${LIBFFI_VERSION}.orig.tar.gz" ${LIBFFI_SHA256}
 tar zxf libffi*.orig.tar.gz
 PATH=/opt/perl/bin:$PATH
@@ -22,7 +22,11 @@
 else
   STACK_PROTECTOR_FLAGS="-fstack-protector-strong"
 fi
-./configure --prefix=/usr/local CFLAGS="-g -O2 $STACK_PROTECTOR_FLAGS -Wformat -Werror=format-security"
+if [ "$2" == "m32" ]; then
+  setarch i386 ./configure --prefix=/usr/local CFLAGS="-m32 -g -O2 $STACK_PROTECTOR_FLAGS -Wformat -Werror=format-security"
+else
+  ./configure --prefix=/usr/local CFLAGS="-g -O2 $STACK_PROTECTOR_FLAGS -Wformat -Werror=format-security"
+fi
 make install
 popd
 rm -rf libffi*
diff --git a/docker/install_openssl.sh b/docker/install_openssl.sh
--- a/docker/install_openssl.sh
+++ b/docker/install_openssl.sh
@@ -13,12 +13,16 @@
     rm "${fname}.sha256"
 }
 
-curl -#O "${OPENSSL_URL}/${OPENSSL_NAME}.tar.gz"
+curl -q -#O "${OPENSSL_URL}/${OPENSSL_NAME}.tar.gz"
 check_sha256sum ${OPENSSL_NAME}.tar.gz ${OPENSSL_SHA256}
 tar zxf ${OPENSSL_NAME}.tar.gz
 PATH=/opt/perl/bin:$PATH
 pushd ${OPENSSL_NAME}
-./config no-comp enable-ec_nistp_64_gcc_128 no-shared no-dynamic-engine --prefix=/usr/local --openssldir=/usr/local
+if [ "$2" == "m32" ]; then
+  setarch i386 ./config no-comp no-shared no-dynamic-engine -m32 --prefix=/usr/local --openssldir=/usr/local
+else
+  ./config no-comp enable-ec_nistp_64_gcc_128 no-shared no-dynamic-engine --prefix=/usr/local --openssldir=/usr/local
+fi
 make depend
 make -j4
 # avoid installing the docs

From pypy.commits at gmail.com  Thu Nov 28 06:17:35 2019
From: pypy.commits at gmail.com (antocuni)
Date: Thu, 28 Nov 2019 03:17:35 -0800 (PST)
Subject: [pypy-commit] pypy hpy: update hpy_universal/_vendored to git
 revision 2129945
Message-ID: <5ddfaccf.1c69fb81.bb0b4.11f2@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98165:988d96254bec
Date: 2019-11-28 01:20 +0100
http://bitbucket.org/pypy/pypy/changeset/988d96254bec/

Log:	update hpy_universal/_vendored to git revision 2129945

diff --git a/pypy/module/hpy_universal/_vendored/include/common/autogen_impl.h b/pypy/module/hpy_universal/_vendored/include/common/autogen_impl.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/_vendored/include/common/autogen_impl.h
@@ -0,0 +1,110 @@
+
+/*
+   DO NOT EDIT THIS FILE!
+
+   This file is automatically generated by tools/autogen.py from tools/public_api.h.
+   Run this to regenerate:
+       make autogen
+
+*/
+
+HPyAPI_STORAGE HPy _HPy_IMPL_NAME(Long_FromLong)(HPyContext ctx, long value)
+{
+    return _py2h(PyLong_FromLong(value));
+}
+
+HPyAPI_STORAGE HPy _HPy_IMPL_NAME(Long_FromLongLong)(HPyContext ctx, long long v)
+{
+    return _py2h(PyLong_FromLongLong(v));
+}
+
+HPyAPI_STORAGE HPy _HPy_IMPL_NAME(Long_FromUnsignedLongLong)(HPyContext ctx, unsigned long long v)
+{
+    return _py2h(PyLong_FromUnsignedLongLong(v));
+}
+
+HPyAPI_STORAGE long _HPy_IMPL_NAME(Long_AsLong)(HPyContext ctx, HPy h)
+{
+    return PyLong_AsLong(_h2py(h));
+}
+
+HPyAPI_STORAGE HPy _HPy_IMPL_NAME(Float_FromDouble)(HPyContext ctx, double v)
+{
+    return _py2h(PyFloat_FromDouble(v));
+}
+
+HPyAPI_STORAGE HPy _HPy_IMPL_NAME(Number_Add)(HPyContext ctx, HPy x, HPy y)
+{
+    return _py2h(PyNumber_Add(_h2py(x), _h2py(y)));
+}
+
+HPyAPI_STORAGE void _HPy_IMPL_NAME(Err_SetString)(HPyContext ctx, HPy type, const char *message)
+{
+    return PyErr_SetString(_h2py(type), message);
+}
+
+HPyAPI_STORAGE int _HPy_IMPL_NAME(Bytes_Check)(HPyContext ctx, HPy o)
+{
+    return PyBytes_Check(_h2py(o));
+}
+
+HPyAPI_STORAGE HPy_ssize_t _HPy_IMPL_NAME(Bytes_Size)(HPyContext ctx, HPy o)
+{
+    return PyBytes_Size(_h2py(o));
+}
+
+HPyAPI_STORAGE HPy_ssize_t _HPy_IMPL_NAME(Bytes_GET_SIZE)(HPyContext ctx, HPy o)
+{
+    return PyBytes_GET_SIZE(_h2py(o));
+}
+
+HPyAPI_STORAGE char *_HPy_IMPL_NAME(Bytes_AsString)(HPyContext ctx, HPy o)
+{
+    return PyBytes_AsString(_h2py(o));
+}
+
+HPyAPI_STORAGE char *_HPy_IMPL_NAME(Bytes_AS_STRING)(HPyContext ctx, HPy o)
+{
+    return PyBytes_AS_STRING(_h2py(o));
+}
+
+HPyAPI_STORAGE HPy _HPy_IMPL_NAME(Unicode_FromString)(HPyContext ctx, const char *utf8)
+{
+    return _py2h(PyUnicode_FromString(utf8));
+}
+
+HPyAPI_STORAGE int _HPy_IMPL_NAME(Unicode_Check)(HPyContext ctx, HPy o)
+{
+    return PyUnicode_Check(_h2py(o));
+}
+
+HPyAPI_STORAGE HPy _HPy_IMPL_NAME(Unicode_AsUTF8String)(HPyContext ctx, HPy o)
+{
+    return _py2h(PyUnicode_AsUTF8String(_h2py(o)));
+}
+
+HPyAPI_STORAGE HPy _HPy_IMPL_NAME(Unicode_FromWideChar)(HPyContext ctx, const wchar_t *w, HPy_ssize_t size)
+{
+    return _py2h(PyUnicode_FromWideChar(w, size));
+}
+
+HPyAPI_STORAGE HPy _HPy_IMPL_NAME(List_New)(HPyContext ctx, HPy_ssize_t len)
+{
+    return _py2h(PyList_New(len));
+}
+
+HPyAPI_STORAGE int _HPy_IMPL_NAME(List_Append)(HPyContext ctx, HPy list, HPy item)
+{
+    return PyList_Append(_h2py(list), _h2py(item));
+}
+
+HPyAPI_STORAGE HPy _HPy_IMPL_NAME(Dict_New)(HPyContext ctx)
+{
+    return _py2h(PyDict_New());
+}
+
+HPyAPI_STORAGE int _HPy_IMPL_NAME(Dict_SetItem)(HPyContext ctx, HPy dict, HPy key, HPy val)
+{
+    return PyDict_SetItem(_h2py(dict), _h2py(key), _h2py(val));
+}
+
diff --git a/pypy/module/hpy_universal/_vendored/include/cpython/hpy.h b/pypy/module/hpy_universal/_vendored/include/cpython/hpy.h
--- a/pypy/module/hpy_universal/_vendored/include/cpython/hpy.h
+++ b/pypy/module/hpy_universal/_vendored/include/cpython/hpy.h
@@ -1,9 +1,6 @@
 #ifndef HPy_CPYTHON_H
 #define HPy_CPYTHON_H
 
-
-
-
 /* XXX: it would be nice if we could include hpy.h WITHOUT bringing in all the
    stuff from Python.h, to make sure that people don't use the CPython API by
    mistake. How to achieve it, though? Is defining Py_LIMITED_API enough? */
@@ -17,11 +14,12 @@
 #include <Python.h>
 
 #ifdef __GNUC__
-#define HPyAPI_FUNC(restype)  __attribute__((unused)) static inline restype
+#define HPyAPI_STORAGE __attribute__((unused)) static inline
 #else
-#define HPyAPI_FUNC(restype)  static inline restype
-#endif
+#define HPyAPI_STORAGE static inline
+#endif /* __GNUC__ */
 
+#define HPyAPI_FUNC(restype) HPyAPI_STORAGE restype
 
 typedef struct { PyObject *_o; } HPy;
 typedef Py_ssize_t HPy_ssize_t;
@@ -33,11 +31,14 @@
 #define _h2py(x) (x._o)
 #define _py2h(o) ((HPy){o})
 
+#include "meth.h"
+
 typedef struct _HPyContext_s {
     HPy h_None;
     HPy h_True;
     HPy h_False;
     HPy h_ValueError;
+    HPy h_TypeError;
 } *HPyContext;
 
 /* XXX! should be defined only once, not once for every .c! */
@@ -46,6 +47,13 @@
 #define HPy_NULL ((HPy){NULL})
 #define HPy_IsNull(x) ((x)._o == NULL)
 
+// XXX: we need to decide whether these are part of the official API or not,
+// and maybe introduce a better naming convetion. For now, they are needed for
+// ujson
+static inline HPy HPy_FromVoidP(void *p) { return (HPy){(PyObject*)p}; }
+static inline void* HPy_AsVoidP(HPy h) { return (void*)h._o; }
+
+
 HPyAPI_FUNC(HPyContext)
 _HPyGetContext(void) {
     HPyContext ctx = &_global_ctx;
@@ -56,19 +64,12 @@
         ctx->h_True = _py2h(Py_True);
         ctx->h_False = _py2h(Py_False);
         ctx->h_ValueError = _py2h(PyExc_ValueError);
+        ctx->h_TypeError = _py2h(PyExc_TypeError);
     }
     return ctx;
 }
 
 
-
-HPyAPI_FUNC(HPy)
-HPyNone_Get(HPyContext ctx)
-{
-    Py_INCREF(Py_None);
-    return _py2h(Py_None);
-}
-
 HPyAPI_FUNC(HPy)
 HPy_Dup(HPyContext ctx, HPy handle)
 {
@@ -84,6 +85,7 @@
 
 /* moduleobject.h */
 typedef PyModuleDef HPyModuleDef;
+
 #define HPyModuleDef_HEAD_INIT PyModuleDef_HEAD_INIT
 
 HPyAPI_FUNC(HPy)
@@ -99,37 +101,13 @@
         return _h2py(init_##modname##_impl(_HPyGetContext())); \
     }
 
-/* methodobject.h */
-typedef PyMethodDef HPyMethodDef;
-
-
-/* function declaration */
-
-#define HPy_METH_NOARGS(NAME)                                           \
-    static HPy NAME##_impl(HPyContext, HPy);                            \
-    static PyObject* NAME(PyObject *self, PyObject *noargs)             \
-    {                                                                   \
-        return _h2py(NAME##_impl(_HPyGetContext(), _py2h(self)));       \
-    }
-
-#define HPy_METH_O(NAME)                                                \
-    static HPy NAME##_impl(HPyContext, HPy, HPy);                       \
-    static PyObject* NAME(PyObject *self, PyObject *arg)                \
-    {                                                                   \
-        return _h2py(NAME##_impl(_HPyGetContext(), _py2h(self), _py2h(arg)));\
-    }
-
-#define HPy_METH_VARARGS(NAME)                                          \
-    static HPy NAME##_impl(HPyContext, HPy, HPy *, Py_ssize_t);         \
-    static PyObject* NAME(PyObject *self, PyObject *args)               \
-    {                                                                   \
-        /* get the tuple elements as an array of "PyObject *", which */ \
-        /* is equivalent to an array of "HPy" with enough casting... */ \
-        HPy *items = (HPy *)&PyTuple_GET_ITEM(args, 0);                 \
-        Py_ssize_t nargs = PyTuple_GET_SIZE(args);                      \
-        return _h2py(NAME##_impl(_HPyGetContext(), _py2h(self), items, nargs));\
-    }
-
+/* XXX: this function is copied&pasted THREE times:
+ *     hpy_devel/include/hpy.h
+ *     cpython-universal/api.c
+ *     pypy/module/hpy_universal/src/getargs.c
+ *
+ * We need a way to share this kind of common code
+ */
 
 HPyAPI_FUNC(int)
 HPyArg_Parse(HPyContext ctx, HPy *args, Py_ssize_t nargs, const char *fmt, ...)
@@ -166,30 +144,6 @@
 
 
 HPyAPI_FUNC(HPy)
-HPyLong_FromLong(HPyContext ctx, long v)
-{
-    return _py2h(PyLong_FromLong(v));
-}
-
-HPyAPI_FUNC(long)
-HPyLong_AsLong(HPyContext ctx, HPy h)
-{
-    return PyLong_AsLong(_h2py(h));
-}
-
-HPyAPI_FUNC(HPy)
-HPyNumber_Add(HPyContext ctx, HPy x, HPy y)
-{
-    return _py2h(PyNumber_Add(_h2py(x), _h2py(y)));
-}
-
-HPyAPI_FUNC(HPy)
-HPyUnicode_FromString(HPyContext ctx, const char *utf8)
-{
-    return _py2h(PyUnicode_FromString(utf8));
-}
-
-HPyAPI_FUNC(HPy)
 HPy_FromPyObject(HPyContext ctx, PyObject *obj)
 {
     Py_XINCREF(obj);
@@ -204,11 +158,12 @@
     return result;
 }
 
-HPyAPI_FUNC(void)
-HPyErr_SetString(HPyContext ctx, HPy type, const char *message)
-{
-    PyErr_SetString(_h2py(type), message);
-}
-
+/* expand impl functions as:
+ *     static inline HPyLong_FromLong(...);
+ *
+ */
+#define _HPy_IMPL_NAME(name) HPy##name
+#include "../common/autogen_impl.h"
+#undef _HPy_IMPL_NAME
 
 #endif /* !HPy_CPYTHON_H */
diff --git a/pypy/module/hpy_universal/_vendored/include/cpython/meth.h b/pypy/module/hpy_universal/_vendored/include/cpython/meth.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/_vendored/include/cpython/meth.h
@@ -0,0 +1,50 @@
+#ifndef HPY_CPYTHON_METH_H
+#define HPY_CPYTHON_METH_H
+
+typedef PyMethodDef HPyMethodDef;
+
+// this is the type of the field HPyMethodDef.ml_meth: cast to it to silence
+// warnings
+typedef PyCFunction HPyMeth;
+
+// XXX: we need to find a way to let the user declare things as static if
+// he/she wants
+
+/* METH declaration */
+#define HPy_DECL_METH_NOARGS(NAME) PyObject* NAME(PyObject *, PyObject *);
+#define HPy_DECL_METH_O(NAME) HPy_DECL_METH_NOARGS(NAME)
+#define HPy_DECL_METH_VARARGS(NAME) HPy_DECL_METH_NOARGS(NAME)
+
+/* METH definition */
+#define HPy_DEF_METH_NOARGS(NAME)                                       \
+    static HPy NAME##_impl(HPyContext, HPy);                            \
+    PyObject* NAME(PyObject *self, PyObject *noargs)                    \
+    {                                                                   \
+        return _h2py(NAME##_impl(_HPyGetContext(), _py2h(self)));       \
+    }
+
+#define HPy_DEF_METH_O(NAME)                                            \
+    static HPy NAME##_impl(HPyContext, HPy, HPy);                       \
+    PyObject* NAME(PyObject *self, PyObject *arg)                       \
+    {                                                                   \
+        return _h2py(NAME##_impl(_HPyGetContext(), _py2h(self), _py2h(arg)));\
+    }
+
+#define HPy_DEF_METH_VARARGS(NAME)                                      \
+    static HPy NAME##_impl(HPyContext, HPy, HPy *, Py_ssize_t);         \
+    PyObject* NAME(PyObject *self, PyObject *args)                      \
+    {                                                                   \
+        /* get the tuple elements as an array of "PyObject *", which */ \
+        /* is equivalent to an array of "HPy" with enough casting... */ \
+        HPy *items = (HPy *)&PyTuple_GET_ITEM(args, 0);                 \
+        Py_ssize_t nargs = PyTuple_GET_SIZE(args);                      \
+        return _h2py(NAME##_impl(_HPyGetContext(), _py2h(self), items, nargs));\
+    }
+
+#define HPy_METH_NOARGS METH_NOARGS
+#define HPy_METH_O METH_O
+#define HPy_METH_VARARGS METH_VARARGS
+
+
+
+#endif // HPY_CPYTHON_METH_H
diff --git a/pypy/module/hpy_universal/_vendored/include/universal/autogen_ctx.h b/pypy/module/hpy_universal/_vendored/include/universal/autogen_ctx.h
--- a/pypy/module/hpy_universal/_vendored/include/universal/autogen_ctx.h
+++ b/pypy/module/hpy_universal/_vendored/include/universal/autogen_ctx.h
@@ -14,15 +14,31 @@
     HPy h_True;
     HPy h_False;
     HPy h_ValueError;
+    HPy h_TypeError;
     HPy (*ctx_Module_Create)(HPyContext ctx, HPyModuleDef *def);
     HPy (*ctx_Dup)(HPyContext ctx, HPy h);
     void (*ctx_Close)(HPyContext ctx, HPy h);
     HPy (*ctx_Long_FromLong)(HPyContext ctx, long value);
+    HPy (*ctx_Long_FromLongLong)(HPyContext ctx, long long v);
+    HPy (*ctx_Long_FromUnsignedLongLong)(HPyContext ctx, unsigned long long v);
     long (*ctx_Long_AsLong)(HPyContext ctx, HPy h);
+    HPy (*ctx_Float_FromDouble)(HPyContext ctx, double v);
     int (*ctx_Arg_Parse)(HPyContext ctx, HPy *args, HPy_ssize_t nargs, const char *fmt, va_list _vl);
     HPy (*ctx_Number_Add)(HPyContext ctx, HPy x, HPy y);
+    void (*ctx_Err_SetString)(HPyContext ctx, HPy type, const char *message);
+    int (*ctx_Bytes_Check)(HPyContext ctx, HPy o);
+    HPy_ssize_t (*ctx_Bytes_Size)(HPyContext ctx, HPy o);
+    HPy_ssize_t (*ctx_Bytes_GET_SIZE)(HPyContext ctx, HPy o);
+    char *(*ctx_Bytes_AsString)(HPyContext ctx, HPy o);
+    char *(*ctx_Bytes_AS_STRING)(HPyContext ctx, HPy o);
     HPy (*ctx_Unicode_FromString)(HPyContext ctx, const char *utf8);
-    void (*ctx_Err_SetString)(HPyContext ctx, HPy type, const char *message);
+    int (*ctx_Unicode_Check)(HPyContext ctx, HPy o);
+    HPy (*ctx_Unicode_AsUTF8String)(HPyContext ctx, HPy o);
+    HPy (*ctx_Unicode_FromWideChar)(HPyContext ctx, const wchar_t *w, HPy_ssize_t size);
+    HPy (*ctx_List_New)(HPyContext ctx, HPy_ssize_t len);
+    int (*ctx_List_Append)(HPyContext ctx, HPy list, HPy item);
+    HPy (*ctx_Dict_New)(HPyContext ctx);
+    int (*ctx_Dict_SetItem)(HPyContext ctx, HPy dict, HPy key, HPy val);
     HPy (*ctx_FromPyObject)(HPyContext ctx, struct _object *obj);
     struct _object *(*ctx_AsPyObject)(HPyContext ctx, HPy h);
     struct _object *(*ctx_CallRealFunctionFromTrampoline)(HPyContext ctx, struct _object *self, struct _object *args, void *func, int ml_flags);
diff --git a/pypy/module/hpy_universal/_vendored/include/universal/autogen_trampolines.h b/pypy/module/hpy_universal/_vendored/include/universal/autogen_trampolines.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/_vendored/include/universal/autogen_trampolines.h
@@ -0,0 +1,118 @@
+
+/*
+   DO NOT EDIT THIS FILE!
+
+   This file is automatically generated by tools/autogen.py from tools/public_api.h.
+   Run this to regenerate:
+       make autogen
+
+*/
+
+static inline HPy HPyModule_Create(HPyContext ctx, HPyModuleDef *def) {
+     return ctx->ctx_Module_Create ( ctx, def ); 
+}
+
+static inline HPy HPy_Dup(HPyContext ctx, HPy h) {
+     return ctx->ctx_Dup ( ctx, h ); 
+}
+
+static inline void HPy_Close(HPyContext ctx, HPy h) {
+     ctx->ctx_Close ( ctx, h ); 
+}
+
+static inline HPy HPyLong_FromLong(HPyContext ctx, long value) {
+     return ctx->ctx_Long_FromLong ( ctx, value ); 
+}
+
+static inline HPy HPyLong_FromLongLong(HPyContext ctx, long long v) {
+     return ctx->ctx_Long_FromLongLong ( ctx, v ); 
+}
+
+static inline HPy HPyLong_FromUnsignedLongLong(HPyContext ctx, unsigned long long v) {
+     return ctx->ctx_Long_FromUnsignedLongLong ( ctx, v ); 
+}
+
+static inline long HPyLong_AsLong(HPyContext ctx, HPy h) {
+     return ctx->ctx_Long_AsLong ( ctx, h ); 
+}
+
+static inline HPy HPyFloat_FromDouble(HPyContext ctx, double v) {
+     return ctx->ctx_Float_FromDouble ( ctx, v ); 
+}
+
+static inline int HPyArg_Parse(HPyContext ctx, HPy *args, HPy_ssize_t nargs, const char *fmt, ...) {
+     va_list _vl; va_start(_vl, fmt); int _res = ctx->ctx_Arg_Parse ( ctx, args, nargs, fmt, _vl ); va_end(_vl); return _res; 
+}
+
+static inline HPy HPyNumber_Add(HPyContext ctx, HPy x, HPy y) {
+     return ctx->ctx_Number_Add ( ctx, x, y ); 
+}
+
+static inline void HPyErr_SetString(HPyContext ctx, HPy type, const char *message) {
+     ctx->ctx_Err_SetString ( ctx, type, message ); 
+}
+
+static inline int HPyBytes_Check(HPyContext ctx, HPy o) {
+     return ctx->ctx_Bytes_Check ( ctx, o ); 
+}
+
+static inline HPy_ssize_t HPyBytes_Size(HPyContext ctx, HPy o) {
+     return ctx->ctx_Bytes_Size ( ctx, o ); 
+}
+
+static inline HPy_ssize_t HPyBytes_GET_SIZE(HPyContext ctx, HPy o) {
+     return ctx->ctx_Bytes_GET_SIZE ( ctx, o ); 
+}
+
+static inline char *HPyBytes_AsString(HPyContext ctx, HPy o) {
+     return ctx->ctx_Bytes_AsString ( ctx, o ); 
+}
+
+static inline char *HPyBytes_AS_STRING(HPyContext ctx, HPy o) {
+     return ctx->ctx_Bytes_AS_STRING ( ctx, o ); 
+}
+
+static inline HPy HPyUnicode_FromString(HPyContext ctx, const char *utf8) {
+     return ctx->ctx_Unicode_FromString ( ctx, utf8 ); 
+}
+
+static inline int HPyUnicode_Check(HPyContext ctx, HPy o) {
+     return ctx->ctx_Unicode_Check ( ctx, o ); 
+}
+
+static inline HPy HPyUnicode_AsUTF8String(HPyContext ctx, HPy o) {
+     return ctx->ctx_Unicode_AsUTF8String ( ctx, o ); 
+}
+
+static inline HPy HPyUnicode_FromWideChar(HPyContext ctx, const wchar_t *w, HPy_ssize_t size) {
+     return ctx->ctx_Unicode_FromWideChar ( ctx, w, size ); 
+}
+
+static inline HPy HPyList_New(HPyContext ctx, HPy_ssize_t len) {
+     return ctx->ctx_List_New ( ctx, len ); 
+}
+
+static inline int HPyList_Append(HPyContext ctx, HPy list, HPy item) {
+     return ctx->ctx_List_Append ( ctx, list, item ); 
+}
+
+static inline HPy HPyDict_New(HPyContext ctx) {
+     return ctx->ctx_Dict_New ( ctx ); 
+}
+
+static inline int HPyDict_SetItem(HPyContext ctx, HPy dict, HPy key, HPy val) {
+     return ctx->ctx_Dict_SetItem ( ctx, dict, key, val ); 
+}
+
+static inline HPy HPy_FromPyObject(HPyContext ctx, struct _object *obj) {
+     return ctx->ctx_FromPyObject ( ctx, obj ); 
+}
+
+static inline struct _object *HPy_AsPyObject(HPyContext ctx, HPy h) {
+     return ctx->ctx_AsPyObject ( ctx, h ); 
+}
+
+static inline struct _object *_HPy_CallRealFunctionFromTrampoline(HPyContext ctx, struct _object *self, struct _object *args, void *func, int ml_flags) {
+     return ctx->ctx_CallRealFunctionFromTrampoline ( ctx, self, args, func, ml_flags ); 
+}
+
diff --git a/pypy/module/hpy_universal/_vendored/include/universal/hpy.h b/pypy/module/hpy_universal/_vendored/include/universal/hpy.h
--- a/pypy/module/hpy_universal/_vendored/include/universal/hpy.h
+++ b/pypy/module/hpy_universal/_vendored/include/universal/hpy.h
@@ -1,113 +1,33 @@
-#ifndef HPy_UNIVERSAL_H
-#define HPy_UNIVERSAL_H
+#ifndef HPY_UNIVERSAL_H
+#define HPY_UNIVERSAL_H
 
 #include <stdlib.h>
 #include <stdint.h>
 #include <stdarg.h>
 
 typedef intptr_t HPy_ssize_t;
-
-// WARNING: the following change has been done inside the pypy hpy-ctypespace
-// branch. If/when the branch is merged, we should backport it to pyhandle/hpy
-struct _HPy_s { HPy_ssize_t _i; };
-typedef struct _HPy_s HPy;
+typedef struct { HPy_ssize_t _i; } HPy;
 
 typedef struct _HPyContext_s *HPyContext;
 struct _object;  /* that's PyObject inside CPython */
 typedef struct _object *(*_HPy_CPyCFunction)(struct _object *self,
                                              struct _object *args);
 
+#define _HPy_HIDDEN   __attribute__((visibility("hidden")))
 #define HPy_NULL ((HPy){0})
 #define HPy_IsNull(x) ((x)._i == 0)
 
-typedef void (*_HPyMethodPairFunc)(void **out_func,
-                                   _HPy_CPyCFunction *out_trampoline);
+// XXX: we need to decide whether these are part of the official API or not,
+// and maybe introduce a better naming convetion. For now, they are needed for
+// ujson
+static inline HPy HPy_FromVoidP(void *p) { return (HPy){(HPy_ssize_t)p}; }
+static inline void* HPy_AsVoidP(HPy h) { return (void*)h._i; }
 
-typedef struct {
-    const char   *ml_name;   /* The name of the built-in function/method */
-    _HPyMethodPairFunc ml_meth;   /* see HPy_FUNCTION() */
-    int          ml_flags;   /* Combination of METH_xxx flags, which mostly
-                                describe the args expected by the C func */
-    const char   *ml_doc;    /* The __doc__ attribute, or NULL */
-} HPyMethodDef;
 
-
-#define HPyModuleDef_HEAD_INIT NULL
-
-typedef struct {
-    void *dummy; // this is needed because we put a comma after HPyModuleDef_HEAD_INIT :(
-    const char* m_name;
-    const char* m_doc;
-    HPy_ssize_t m_size;
-    HPyMethodDef *m_methods;
-} HPyModuleDef;
-
-#define _HPy_HIDDEN   __attribute__((visibility("hidden")))
-#define HPy_MODINIT(modname)                                   \
-    _HPy_HIDDEN HPyContext _ctx_for_trampolines;               \
-    static HPy init_##modname##_impl(HPyContext ctx);          \
-    HPy HPyInit_##modname(HPyContext ctx)                      \
-    {                                                          \
-        _ctx_for_trampolines = ctx;                            \
-        return init_##modname##_impl(ctx);                     \
-    }
+#include "meth.h"
+#include "module.h"
 
 #include "autogen_ctx.h"
-#include "autogen_func.h"
+#include "autogen_trampolines.h"
 
-extern HPyContext _ctx_for_trampolines;
-
-
-#define HPy_METH_NOARGS(fnname)                                                \
-    static HPy fnname##_impl(HPyContext ctx, HPy self);                        \
-    static struct _object *                                                    \
-    fnname##_trampoline(struct _object *self, struct _object *noargs)          \
-    {                                                                          \
-        return _HPy_CallRealFunctionFromTrampoline(                            \
-            _ctx_for_trampolines, self, NULL, fnname##_impl, METH_NOARGS);     \
-    }                                                                          \
-    static void                                                                \
-    fnname(void **out_func, _HPy_CPyCFunction *out_trampoline)                 \
-    {                                                                          \
-        *out_func = fnname##_impl;                                             \
-        *out_trampoline = fnname##_trampoline;                                 \
-    }
-
-#define HPy_METH_O(fnname)                                                     \
-    static HPy fnname##_impl(HPyContext ctx, HPy self, HPy arg);               \
-    static struct _object *                                                    \
-    fnname##_trampoline(struct _object *self, struct _object *arg)             \
-    {                                                                          \
-        return _HPy_CallRealFunctionFromTrampoline(                            \
-            _ctx_for_trampolines, self, arg, fnname##_impl, METH_O);           \
-    }                                                                          \
-    static void                                                                \
-    fnname(void **out_func, _HPy_CPyCFunction *out_trampoline)                 \
-    {                                                                          \
-        *out_func = fnname##_impl;                                             \
-        *out_trampoline = fnname##_trampoline;                                 \
-    }
-
-#define HPy_METH_VARARGS(fnname)                                               \
-    static HPy fnname##_impl(HPyContext ctx, HPy self, HPy *args, HPy_ssize_t);\
-    static struct _object *                                                    \
-    fnname##_trampoline(struct _object *self, struct _object *args)            \
-    {                                                                          \
-        return _HPy_CallRealFunctionFromTrampoline(                            \
-            _ctx_for_trampolines, self, args, fnname##_impl, METH_VARARGS);    \
-    }                                                                          \
-    static void                                                                \
-    fnname(void **out_func, _HPy_CPyCFunction *out_trampoline)                 \
-    {                                                                          \
-        *out_func = fnname##_impl;                                             \
-        *out_trampoline = fnname##_trampoline;                                 \
-    }
-
-#define METH_VARARGS  0x0001
-#define METH_KEYWORDS 0x0002
-/* METH_NOARGS and METH_O must not be combined with the flags above. */
-#define METH_NOARGS   0x0004
-#define METH_O        0x0008
-
-
-#endif /* HPy_UNIVERSAL_H */
+#endif /* HPY_UNIVERSAL_H */
diff --git a/pypy/module/hpy_universal/_vendored/include/universal/meth.h b/pypy/module/hpy_universal/_vendored/include/universal/meth.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/_vendored/include/universal/meth.h
@@ -0,0 +1,95 @@
+#ifndef HPY_UNIVERSAL_METH_H
+#define HPY_UNIVERSAL_METH_H
+
+/* in universal mode, an HPyMeth is a function which returns two output
+ * arguments:
+ *
+ *     - the *_impl function
+ *     - a trampoline which can be called by CPython
+ *
+ * In theory, the CPython trampoline is an implementation-specific detail of
+ * the hpy_universal CPython module. However, it is too hard and unreliable to
+ * generate them on the fly, and for the sake of simplicity it is easier to
+ * just let the C compiler to generate it. This is done by the DEF macros.
+ */
+typedef void (*HPyMeth)(void **out_func, _HPy_CPyCFunction *out_trampoline);
+
+/* ml_flags can be:
+ *
+ *   - METH_NOARGS, METH_O, etc: in this case ml_meth is interpreted as a legacy
+ *     CPython function
+ *
+ *   - HPy_METH_NOARGS, etc: in this case ml_meth is interpreted as a new-style
+ *     HPy function
+ */
+typedef struct {
+    const char   *ml_name;   /* The name of the built-in function/method */
+    HPyMeth      ml_meth;    /* see HPy_DEF_METH_*() */
+    int          ml_flags;   /* Combination of METH_xxx flags, which mostly
+                                describe the args expected by the C func */
+    const char   *ml_doc;    /* The __doc__ attribute, or NULL */
+} HPyMethodDef;
+
+#define HPy_DECL_METH_NOARGS(fnname)                                    \
+    void fnname(void **out_func, _HPy_CPyCFunction *out_trampoline);
+
+#define HPy_DECL_METH_O(NAME) HPy_DECL_METH_NOARGS(NAME)
+#define HPy_DECL_METH_VARARGS(NAME) HPy_DECL_METH_NOARGS(NAME)
+
+
+#define HPy_DEF_METH_NOARGS(fnname)                                            \
+    static HPy fnname##_impl(HPyContext ctx, HPy self);                        \
+    static struct _object *                                                    \
+    fnname##_trampoline(struct _object *self, struct _object *noargs)          \
+    {                                                                          \
+        return _HPy_CallRealFunctionFromTrampoline(                            \
+            _ctx_for_trampolines, self, NULL, fnname##_impl, HPy_METH_NOARGS); \
+    }                                                                          \
+    void                                                                       \
+    fnname(void **out_func, _HPy_CPyCFunction *out_trampoline)                 \
+    {                                                                          \
+        *out_func = fnname##_impl;                                             \
+        *out_trampoline = fnname##_trampoline;                                 \
+    }
+
+#define HPy_DEF_METH_O(fnname)                                                 \
+    static HPy fnname##_impl(HPyContext ctx, HPy self, HPy arg);               \
+    static struct _object *                                                    \
+    fnname##_trampoline(struct _object *self, struct _object *arg)             \
+    {                                                                          \
+        return _HPy_CallRealFunctionFromTrampoline(                            \
+            _ctx_for_trampolines, self, arg, fnname##_impl, HPy_METH_O);       \
+    }                                                                          \
+    void                                                                       \
+    fnname(void **out_func, _HPy_CPyCFunction *out_trampoline)                 \
+    {                                                                          \
+        *out_func = fnname##_impl;                                             \
+        *out_trampoline = fnname##_trampoline;                                 \
+    }
+
+#define HPy_DEF_METH_VARARGS(fnname)                                           \
+    static HPy fnname##_impl(HPyContext ctx, HPy self, HPy *args, HPy_ssize_t);\
+    static struct _object *                                                    \
+    fnname##_trampoline(struct _object *self, struct _object *args)            \
+    {                                                                          \
+        return _HPy_CallRealFunctionFromTrampoline(                            \
+            _ctx_for_trampolines, self, args, fnname##_impl, HPy_METH_VARARGS);\
+    }                                                                          \
+    void                                                                       \
+    fnname(void **out_func, _HPy_CPyCFunction *out_trampoline)                 \
+    {                                                                          \
+        *out_func = fnname##_impl;                                             \
+        *out_trampoline = fnname##_trampoline;                                 \
+    }
+
+
+// make sure to use a bit which is unused by CPython
+#define _HPy_METH 0x100000
+#define HPy_METH_VARARGS  (0x0001 | _HPy_METH)
+#define HPy_METH_KEYWORDS (0x0002 | _HPy_METH)
+/* METH_NOARGS and METH_O must not be combined with the flags above. */
+#define HPy_METH_NOARGS   (0x0004 | _HPy_METH)
+#define HPy_METH_O        (0x0008 | _HPy_METH)
+
+
+#endif // HPY_UNIVERSAL_METH_H
diff --git a/pypy/module/hpy_universal/_vendored/include/universal/module.h b/pypy/module/hpy_universal/_vendored/include/universal/module.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/_vendored/include/universal/module.h
@@ -0,0 +1,27 @@
+#ifndef HPY_UNIVERSAL_MODULE_H
+#define HPY_UNIVERSAL_MODULE_H
+
+// this is defined by HPy_MODINIT
+extern HPyContext _ctx_for_trampolines;
+
+#define HPyModuleDef_HEAD_INIT NULL
+
+typedef struct {
+    void *dummy; // this is needed because we put a comma after HPyModuleDef_HEAD_INIT :(
+    const char* m_name;
+    const char* m_doc;
+    HPy_ssize_t m_size;
+    HPyMethodDef *m_methods;
+} HPyModuleDef;
+
+
+#define HPy_MODINIT(modname)                                   \
+    _HPy_HIDDEN HPyContext _ctx_for_trampolines;               \
+    static HPy init_##modname##_impl(HPyContext ctx);          \
+    HPy HPyInit_##modname(HPyContext ctx)                      \
+    {                                                          \
+        _ctx_for_trampolines = ctx;                            \
+        return init_##modname##_impl(ctx);                     \
+    }
+
+#endif // HPY_UNIVERSAL_MODULE_H
diff --git a/pypy/module/hpy_universal/_vendored/test/conftest.py b/pypy/module/hpy_universal/_vendored/test/conftest.py
--- a/pypy/module/hpy_universal/_vendored/test/conftest.py
+++ b/pypy/module/hpy_universal/_vendored/test/conftest.py
@@ -6,6 +6,10 @@
         "--correct", action="store_true",
         help="Test against headers installed through hpy_devel"
     )
+    parser.addoption(
+        "--compiler-v", action="store_true",
+        help="Print to stdout the commands used to invoke the compiler")
+
 
 @pytest.fixture(scope='session')
 def hpy_include_dir(request):
@@ -22,5 +26,7 @@
     return request.param
 
 @pytest.fixture
-def compiler(tmpdir, abimode, hpy_include_dir):
-    return ExtensionCompiler(tmpdir, abimode, hpy_include_dir)
+def compiler(request, tmpdir, abimode, hpy_include_dir):
+    compiler_verbose = request.config.getoption('--compiler-v')
+    return ExtensionCompiler(tmpdir, abimode, hpy_include_dir,
+                             compiler_verbose=compiler_verbose)
diff --git a/pypy/module/hpy_universal/_vendored/test/support.py b/pypy/module/hpy_universal/_vendored/test/support.py
--- a/pypy/module/hpy_universal/_vendored/test/support.py
+++ b/pypy/module/hpy_universal/_vendored/test/support.py
@@ -3,7 +3,7 @@
 import re
 
 r_marker_init = re.compile(r"\s*@INIT\s*$")
-r_marker_export = re.compile(r"\s*@EXPORT\s+(\w+)\s+(METH_\w+)\s*$")
+r_marker_export = re.compile(r"\s*@EXPORT\s+(\w+)\s+(.*)\s*$")
 
 INIT_TEMPLATE = """
 static HPyMethodDef MyTestMethods[] = {
@@ -47,8 +47,14 @@
         match = r_marker_export.match(line)
         if match:
             ml_name, ml_flags = match.group(1), match.group(2)
-            method_table.append('{"%s", %s, %s, NULL},' % (
-                ml_name, ml_name, ml_flags))
+            if not ml_flags.startswith('HPy_'):
+                # this is a legacy function: add a cast to (HPyMeth) to
+                # silence warnings
+                cast = '(HPyMeth)'
+            else:
+                cast = ''
+            method_table.append('{"%s", %s%s, %s, NULL},' % (
+                    ml_name, cast, ml_name, ml_flags))
             continue
 
         expanded_lines.append(line)
@@ -62,34 +68,45 @@
 
 
 class ExtensionCompiler:
-    def __init__(self, tmpdir, abimode, include_dir):
+    def __init__(self, tmpdir, abimode, include_dir, compiler_verbose=False):
         self.tmpdir = tmpdir
         self.abimode = abimode
         self.include_dir = include_dir
         self.universal_mode = self.abimode == 'universal'
+        self.compiler_verbose = compiler_verbose
 
-    def compile_module(self, source_template, name):
+    def _expand(self, name, template):
+        source = expand_template(template, name)
+        filename = self.tmpdir.join(name + '.c')
+        filename.write(source)
+        return filename
+
+    def compile_module(self, main_template, name, extra_templates):
         """
         Create and compile a HPy module from the template
         """
-        source = expand_template(source_template, name)
-        filename = self.tmpdir.join(name + '.c')
-        filename.write(source)
+        filename = self._expand(name, main_template)
+        sources = []
+        for i, template in enumerate(extra_templates):
+            extra_filename = self._expand('extmod_%d' % i, template)
+            sources.append(extra_filename)
         #
         ext = get_extension(str(filename), name,
+                            sources=sources,
                             include_dirs=[self.include_dir],
                             extra_compile_args=['-Wfatal-errors', '-g', '-Og'],
                             extra_link_args=['-g'])
-        so_filename = c_compile(str(self.tmpdir), ext, compiler_verbose=False,
+        so_filename = c_compile(str(self.tmpdir), ext,
+                                compiler_verbose=self.compiler_verbose,
                                 universal_mode=self.universal_mode)
         return so_filename
 
-    def make_module(self, source_template, name):
+    def make_module(self, main_template, name, extra_templates):
         """
         Compile&load a modulo into memory. This is NOT a proper import: e.g. the module
         is not put into sys.modules
         """
-        so_filename = self.compile_module(source_template, name)
+        so_filename = self.compile_module(main_template, name, extra_templates)
         if self.universal_mode:
             return self.load_universal_module(name, so_filename)
         else:
@@ -122,11 +139,8 @@
         # compiler is a fixture defined in conftest
         self.compiler = compiler
 
-    def compile_module(self, source_template, name):
-        return self.compiler.compile_module(source_template, name)
-
-    def make_module(self, source_template, name='mytest'):
-        return self.compiler.make_module(source_template, name)
+    def make_module(self, source_template, name='mytest', extra_templates=()):
+        return self.compiler.make_module(source_template, name, extra_templates)
 
 
 # the few functions below are copied and adapted from cffi/ffiplatform.py
diff --git a/pypy/module/hpy_universal/_vendored/test/test_basic.py b/pypy/module/hpy_universal/_vendored/test/test_basic.py
--- a/pypy/module/hpy_universal/_vendored/test/test_basic.py
+++ b/pypy/module/hpy_universal/_vendored/test/test_basic.py
@@ -26,36 +26,36 @@
 
     def test_noop_function(self):
         mod = self.make_module("""
-            HPy_METH_NOARGS(f)
+            HPy_DEF_METH_NOARGS(f)
             static HPy f_impl(HPyContext ctx, HPy self)
             {
                 return HPy_Dup(ctx, ctx->h_None);
             }
-            @EXPORT f METH_NOARGS
+            @EXPORT f HPy_METH_NOARGS
             @INIT
         """)
         assert mod.f() is None
 
     def test_self_is_module(self):
         mod = self.make_module("""
-            HPy_METH_NOARGS(f)
+            HPy_DEF_METH_NOARGS(f)
             static HPy f_impl(HPyContext ctx, HPy self)
             {
                 return HPy_Dup(ctx, self);
             }
-            @EXPORT f METH_NOARGS
+            @EXPORT f HPy_METH_NOARGS
             @INIT
         """)
         assert mod.f() is mod
 
     def test_identity_function(self):
         mod = self.make_module("""
-            HPy_METH_O(f)
+            HPy_DEF_METH_O(f)
             static HPy f_impl(HPyContext ctx, HPy self, HPy arg)
             {
                 return HPy_Dup(ctx, arg);
             }
-            @EXPORT f METH_O
+            @EXPORT f HPy_METH_O
             @INIT
         """)
         x = object()
@@ -63,13 +63,13 @@
 
     def test_long_aslong(self):
         mod = self.make_module("""
-            HPy_METH_O(f)
+            HPy_DEF_METH_O(f)
             static HPy f_impl(HPyContext ctx, HPy self, HPy arg)
             {
                 long a = HPyLong_AsLong(ctx, arg);
                 return HPyLong_FromLong(ctx, a * 2);
             }
-            @EXPORT f METH_O
+            @EXPORT f HPy_METH_O
             @INIT
         """)
         assert mod.f(45) == 90
@@ -77,18 +77,18 @@
     def test_wrong_number_of_arguments(self):
         import pytest
         mod = self.make_module("""
-            HPy_METH_NOARGS(f_noargs)
+            HPy_DEF_METH_NOARGS(f_noargs)
             static HPy f_noargs_impl(HPyContext ctx, HPy self)
             {
                 return HPy_Dup(ctx, ctx->h_None);
             }
-            HPy_METH_O(f_o)
+            HPy_DEF_METH_O(f_o)
             static HPy f_o_impl(HPyContext ctx, HPy self, HPy arg)
             {
                 return HPy_Dup(ctx, ctx->h_None);
             }
-            @EXPORT f_noargs METH_NOARGS
-            @EXPORT f_o METH_O
+            @EXPORT f_noargs HPy_METH_NOARGS
+            @EXPORT f_o HPy_METH_O
             @INIT
         """)
         with pytest.raises(TypeError):
@@ -100,7 +100,7 @@
 
     def test_many_int_arguments(self):
         mod = self.make_module("""
-            HPy_METH_VARARGS(f)
+            HPy_DEF_METH_VARARGS(f)
             static HPy f_impl(HPyContext ctx, HPy self,
                               HPy *args, HPy_ssize_t nargs)
             {
@@ -111,14 +111,14 @@
                 return HPyLong_FromLong(ctx,
                     10000*a + 1000*b + 100*c + 10*d + e);
             }
-            @EXPORT f METH_VARARGS
+            @EXPORT f HPy_METH_VARARGS
             @INIT
         """)
         assert mod.f(4, 5, 6, 7, 8) == 45678
 
     def test_close(self):
         mod = self.make_module("""
-            HPy_METH_O(f)
+            HPy_DEF_METH_O(f)
             static HPy f_impl(HPyContext ctx, HPy self, HPy arg)
             {
                 HPy one = HPyLong_FromLong(ctx, 1);
@@ -128,32 +128,20 @@
                 HPy_Close(ctx, one);
                 return res;
             }
-            @EXPORT f METH_O
+            @EXPORT f HPy_METH_O
             @INIT
         """)
         assert mod.f(41.5) == 42.5
 
-    def test_string(self):
-        mod = self.make_module("""
-            HPy_METH_NOARGS(f)
-            static HPy f_impl(HPyContext ctx, HPy self)
-            {
-                return HPyUnicode_FromString(ctx, "foobar");
-            }
-            @EXPORT f METH_NOARGS
-            @INIT
-        """)
-        assert mod.f() == "foobar"
-
     def test_bool(self):
         mod = self.make_module("""
-            HPy_METH_O(f)
+            HPy_DEF_METH_O(f)
             static HPy f_impl(HPyContext ctx, HPy self, HPy arg)
             {
                 int cond = HPyLong_AsLong(ctx, arg) > 5;
                 return HPy_Dup(ctx, cond ? ctx->h_True : ctx->h_False);
             }
-            @EXPORT f METH_O
+            @EXPORT f HPy_METH_O
             @INIT
         """)
         assert mod.f(4) is False
@@ -162,7 +150,7 @@
     def test_exception(self):
         import pytest
         mod = self.make_module("""
-            HPy_METH_O(f)
+            HPy_DEF_METH_O(f)
             static HPy f_impl(HPyContext ctx, HPy self, HPy arg)
             {
                 long x = HPyLong_AsLong(ctx, arg);
@@ -174,10 +162,115 @@
                     return HPy_NULL;
                 }
             }
-            @EXPORT f METH_O
+            @EXPORT f HPy_METH_O
             @INIT
         """)
         assert mod.f(-10) == 10
         with pytest.raises(ValueError) as exc:
             mod.f(20)
         assert str(exc.value) == 'hello world'
+
+    def test_builtin_handles(self):
+        import pytest
+        mod = self.make_module("""
+            HPy_DEF_METH_O(f)
+            static HPy f_impl(HPyContext ctx, HPy self, HPy arg)
+            {
+                long i = HPyLong_AsLong(ctx, arg);
+                HPy h;
+                switch(i) {
+                    case 1: h = ctx->h_None; break;
+                    case 2: h = ctx->h_False; break;
+                    case 3: h = ctx->h_True; break;
+                    case 4: h = ctx->h_ValueError; break;
+                    case 5: h = ctx->h_TypeError; break;
+                    default:
+                        HPyErr_SetString(ctx, ctx->h_ValueError, "invalid choice");
+                        return HPy_NULL;
+                }
+                return HPy_Dup(ctx, h);
+            }
+            @EXPORT f HPy_METH_O
+            @INIT
+        """)
+        builtin_objs = ('<NULL>', None, False, True, ValueError, TypeError)
+        for i, obj in enumerate(builtin_objs):
+            if i == 0:
+                continue
+            assert mod.f(i) is obj
+
+    def test_decl_meth(self):
+        main = """
+            HPy_DECL_METH_NOARGS(f);
+            HPy_DECL_METH_O(g);
+            HPy_DECL_METH_VARARGS(h);
+
+            @EXPORT f HPy_METH_NOARGS
+            @EXPORT g HPy_METH_O
+            @EXPORT h HPy_METH_VARARGS
+            @INIT
+        """
+        extra = """
+            HPy_DEF_METH_NOARGS(f)
+            static HPy f_impl(HPyContext ctx, HPy self)
+            {
+                return HPyLong_FromLong(ctx, 12345);
+            }
+            HPy_DEF_METH_O(g)
+            static HPy g_impl(HPyContext ctx, HPy self, HPy arg)
+            {
+                return HPy_Dup(ctx, arg);
+            }
+            HPy_DEF_METH_VARARGS(h)
+            static HPy h_impl(HPyContext ctx, HPy self, HPy *args, HPy_ssize_t nargs)
+            {
+                long a, b;
+                if (!HPyArg_Parse(ctx, args, nargs, "ll", &a, &b))
+                    return HPy_NULL;
+                return HPyLong_FromLong(ctx, 10*a + b);
+            }
+        """
+        mod = self.make_module(main, extra_templates=[extra])
+        assert mod.f() == 12345
+        assert mod.g(42) == 42
+        assert mod.h(5, 6) == 56
+
+    def test_Float_FromDouble(self):
+        mod = self.make_module("""
+            HPy_DEF_METH_NOARGS(f)
+            static HPy f_impl(HPyContext ctx, HPy self)
+            {
+                return HPyFloat_FromDouble(ctx, 123.45);
+            }
+            @EXPORT f HPy_METH_NOARGS
+            @INIT
+        """)
+        assert mod.f() == 123.45
+
+    def test_Long_FromLongLong(self):
+        mod = self.make_module("""
+            HPy_DEF_METH_NOARGS(f)
+            static HPy f_impl(HPyContext ctx, HPy self)
+            {
+                // take a value which doesn't fit in 32 bit
+                long long val = 2147483648;
+                return HPyLong_FromLongLong(ctx, val);
+            }
+            @EXPORT f HPy_METH_NOARGS
+            @INIT
+        """)
+        assert mod.f() == 2147483648
+
+    def test_Long_FromUnsignedLongLong(self):
+        mod = self.make_module("""
+            HPy_DEF_METH_NOARGS(f)
+            static HPy f_impl(HPyContext ctx, HPy self)
+            {
+                // take a value which doesn't fit in unsigned 32 bit
+                unsigned long long val = 4294967296;
+                return HPyLong_FromUnsignedLongLong(ctx, val);
+            }
+            @EXPORT f HPy_METH_NOARGS
+            @INIT
+        """)
+        assert mod.f() == 4294967296
diff --git a/pypy/module/hpy_universal/_vendored/test/test_bytesobject.py b/pypy/module/hpy_universal/_vendored/test/test_bytesobject.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/_vendored/test/test_bytesobject.py
@@ -0,0 +1,66 @@
+from .support import HPyTest
+
+class TestBytesObject(HPyTest):
+
+    def test_Check(self):
+        mod = self.make_module("""
+            HPy_DEF_METH_O(f)
+            static HPy f_impl(HPyContext ctx, HPy self, HPy arg)
+            {
+                if (HPyBytes_Check(ctx, arg))
+                    return HPy_Dup(ctx, ctx->h_True);
+                return HPy_Dup(ctx, ctx->h_False);
+            }
+            @EXPORT f HPy_METH_O
+            @INIT
+        """)
+        assert mod.f(b'hello') is True
+        assert mod.f('hello') is False
+
+    def test_Size(self):
+        mod = self.make_module("""
+            HPy_DEF_METH_O(f)
+            static HPy f_impl(HPyContext ctx, HPy self, HPy arg)
+            {
+                HPy_ssize_t a = HPyBytes_Size(ctx, arg);
+                HPy_ssize_t b = HPyBytes_GET_SIZE(ctx, arg);
+                return HPyLong_FromLong(ctx, 10*a + b);
+            }
+            @EXPORT f HPy_METH_O
+            @INIT
+        """)
+        assert mod.f(b'hello') == 55
+
+    def test_AsString(self):
+        mod = self.make_module("""
+            HPy_DEF_METH_O(f)
+            static HPy f_impl(HPyContext ctx, HPy self, HPy arg)
+            {
+                long res = 0;
+                HPy_ssize_t n = HPyBytes_Size(ctx, arg);
+                char *buf = HPyBytes_AsString(ctx, arg);
+                for(int i=0; i<n; i++)
+                    res = (res * 10) + buf[i];
+                return HPyLong_FromLong(ctx, res);
+            }
+            @EXPORT f HPy_METH_O
+            @INIT
+        """)
+        assert mod.f(b'ABC') == 100*ord('A') + 10*ord('B') + ord('C')
+
+    def test_AS_STRING(self):
+        mod = self.make_module("""
+            HPy_DEF_METH_O(f)
+            static HPy f_impl(HPyContext ctx, HPy self, HPy arg)
+            {
+                long res = 0;
+                HPy_ssize_t n = HPyBytes_Size(ctx, arg);
+                char *buf = HPyBytes_AS_STRING(ctx, arg);
+                for(int i=0; i<n; i++)
+                    res = (res * 10) + buf[i];
+                return HPyLong_FromLong(ctx, res);
+            }
+            @EXPORT f HPy_METH_O
+            @INIT
+        """)
+        assert mod.f(b'ABC') == 100*ord('A') + 10*ord('B') + ord('C')
diff --git a/pypy/module/hpy_universal/_vendored/test/test_cpy_compat.py b/pypy/module/hpy_universal/_vendored/test/test_cpy_compat.py
--- a/pypy/module/hpy_universal/_vendored/test/test_cpy_compat.py
+++ b/pypy/module/hpy_universal/_vendored/test/test_cpy_compat.py
@@ -6,7 +6,7 @@
     def test_frompyobject(self):
         mod = self.make_module("""
             #include <Python.h>
-            HPy_METH_NOARGS(f)
+            HPy_DEF_METH_NOARGS(f)
             static HPy f_impl(HPyContext ctx, HPy self)
             {
                 PyObject *o = PyList_New(0);
@@ -20,7 +20,7 @@
                 Py_DECREF(o);
                 return h;
             }
-            @EXPORT f METH_NOARGS
+            @EXPORT f HPy_METH_NOARGS
             @INIT
         """)
         x = mod.f()
@@ -29,7 +29,7 @@
     def test_hpy_close(self):
         mod = self.make_module("""
             #include <Python.h>
-            HPy_METH_NOARGS(f)
+            HPy_DEF_METH_NOARGS(f)
             static HPy f_impl(HPyContext ctx, HPy self)
             {
                 PyObject *o = PyList_New(0);
@@ -43,7 +43,7 @@
                 return HPyLong_FromLong(ctx, (long)(final_refcount -
                                                     initial_refcount));
             }
-            @EXPORT f METH_NOARGS
+            @EXPORT f HPy_METH_NOARGS
             @INIT
         """)
         assert mod.f() == -1
@@ -51,7 +51,7 @@
     def test_hpy_dup(self):
         mod = self.make_module("""
             #include <Python.h>
-            HPy_METH_NOARGS(f)
+            HPy_DEF_METH_NOARGS(f)
             static HPy f_impl(HPyContext ctx, HPy self)
             {
                 PyObject *o = PyList_New(0);
@@ -67,7 +67,7 @@
                 return HPyLong_FromLong(ctx, (long)(final_refcount -
                                                     initial_refcount));
             }
-            @EXPORT f METH_NOARGS
+            @EXPORT f HPy_METH_NOARGS
             @INIT
         """)
         assert mod.f() == +1
@@ -77,7 +77,7 @@
             #include <Python.h>
             #define NUM_HANDLES  10000
 
-            HPy_METH_NOARGS(f)
+            HPy_DEF_METH_NOARGS(f)
             static HPy f_impl(HPyContext ctx, HPy self)
             {
                 PyObject *o = PyList_New(0);
@@ -99,7 +99,67 @@
              error:
                 return HPyLong_FromLong(ctx, (long)result);
             }
+            @EXPORT f HPy_METH_NOARGS
+            @INIT
+        """)
+        assert mod.f() == 0
+
+    def test_meth_cpy_noargs(self):
+        mod = self.make_module("""
+            #include <Python.h>
+
+            static PyObject *f(PyObject *self, PyObject *args)
+            {
+                return PyLong_FromLong(1234);
+            }
             @EXPORT f METH_NOARGS
             @INIT
         """)
-        assert mod.f() == 0
+        assert mod.f() == 1234
+
+    def test_meth_cpy_o(self):
+        mod = self.make_module("""
+            #include <Python.h>
+
+            static PyObject *f(PyObject *self, PyObject *arg)
+            {
+                long x = PyLong_AsLong(arg);
+                return PyLong_FromLong(x * 2);
+            }
+            @EXPORT f METH_O
+            @INIT
+        """)
+        assert mod.f(45) == 90
+
+    def test_meth_cpy_varargs(self):
+        mod = self.make_module("""
+            #include <Python.h>
+
+            static PyObject *f(PyObject *self, PyObject *args)
+            {
+                long a, b, c;
+                if (!PyArg_ParseTuple(args, "lll", &a, &b, &c))
+                    return NULL;
+                return PyLong_FromLong(100*a + 10*b + c);
+            }
+            @EXPORT f METH_VARARGS
+            @INIT
+        """)
+        assert mod.f(4, 5, 6) == 456
+
+    def test_meth_cpy_keywords(self):
+        mod = self.make_module("""
+            #include <Python.h>
+
+            static PyObject *f(PyObject *self, PyObject *args, PyObject *kwargs)
+            {
+                static char *kwlist[] = { "a", "b", "c", NULL };
+                long a, b, c;
+                if (!PyArg_ParseTupleAndKeywords(args, kwargs, "lll", kwlist, &a, &b, &c))
+                    return NULL;
+                return PyLong_FromLong(100*a + 10*b + c);
+            }
+            @EXPORT f METH_VARARGS | METH_KEYWORDS
+            @INIT
+        """)
+        assert mod.f(c=6, b=5, a=4) == 456
diff --git a/pypy/module/hpy_universal/_vendored/test/test_dictobject.py b/pypy/module/hpy_universal/_vendored/test/test_dictobject.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/_vendored/test/test_dictobject.py
@@ -0,0 +1,33 @@
+from .support import HPyTest
+
+class TestDictObject(HPyTest):
+
+    def test_New(self):
+        mod = self.make_module("""
+            HPy_DEF_METH_NOARGS(f)
+            static HPy f_impl(HPyContext ctx, HPy self)
+            {
+                return HPyDict_New(ctx);
+            }
+            @EXPORT f HPy_METH_NOARGS
+            @INIT
+        """)
+        assert mod.f() == {}
+
+    def test_SetItem(self):
+        mod = self.make_module("""
+            HPy_DEF_METH_O(f)
+            static HPy f_impl(HPyContext ctx, HPy self, HPy arg)
+            {
+                HPy dict = HPyDict_New(ctx);
+                if (HPy_IsNull(dict))
+                    return HPy_NULL;
+                HPy val = HPyLong_FromLong(ctx, 1234);
+                if (HPyDict_SetItem(ctx, dict, arg, val) == -1)
+                    return HPy_NULL;
+                return dict;
+            }
+            @EXPORT f HPy_METH_O
+            @INIT
+        """)
+        assert mod.f('hello') == {'hello': 1234}
diff --git a/pypy/module/hpy_universal/_vendored/test/test_listobject.py b/pypy/module/hpy_universal/_vendored/test/test_listobject.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/_vendored/test/test_listobject.py
@@ -0,0 +1,34 @@
+from .support import HPyTest
+
+class TestListObject(HPyTest):
+
+    def test_New(self):
+        mod = self.make_module("""
+            HPy_DEF_METH_NOARGS(f)
+            static HPy f_impl(HPyContext ctx, HPy self)
+            {
+                return HPyList_New(ctx, 0);
+            }
+            @EXPORT f HPy_METH_NOARGS
+            @INIT
+        """)
+        assert mod.f() == []
+
+    def test_Append(self):
+        mod = self.make_module("""
+            HPy_DEF_METH_O(f)
+            static HPy f_impl(HPyContext ctx, HPy self, HPy arg)
+            {
+                HPy list = HPyList_New(ctx, 0);
+                if (HPy_IsNull(list))
+                    return HPy_NULL;
+                if (HPyList_Append(ctx, list, arg) == -1)
+                    return HPy_NULL;
+                if (HPyList_Append(ctx, list, arg) == -1)
+                    return HPy_NULL;
+                return list;
+            }
+            @EXPORT f HPy_METH_O
+            @INIT
+        """)
+        assert mod.f(42) == [42, 42]
diff --git a/pypy/module/hpy_universal/_vendored/test/test_support.py b/pypy/module/hpy_universal/_vendored/test/test_support.py
--- a/pypy/module/hpy_universal/_vendored/test/test_support.py
+++ b/pypy/module/hpy_universal/_vendored/test/test_support.py
@@ -1,13 +1,19 @@
 from . import support
 
-
 def test_expand_template():
     expanded = support.expand_template("""
-        @EXPORT test_f METH_O
+        @EXPORT test_f HPy_METH_NOARGS
+        @EXPORT test_g METH_O
+        @EXPORT test_h METH_VARARGS | METH_KEYWORDS
         some more C stuff
         @INIT
     """, name='mytest')
-    methods = '{"test_f", test_f, METH_O, NULL},'
+    method_table = [
+        '{"test_f", test_f, HPy_METH_NOARGS, NULL},',
+        '{"test_g", (HPyMeth)test_g, METH_O, NULL},',
+        '{"test_h", (HPyMeth)test_h, METH_VARARGS | METH_KEYWORDS, NULL},'
+        ]
+    methods = '\n    '.join(method_table)
     init_code = support.INIT_TEMPLATE % {'methods': methods, 'name': 'mytest'}
     assert expanded.rstrip() == f"""#include <hpy.h>
 
diff --git a/pypy/module/hpy_universal/_vendored/test/test_unicodeobject.py b/pypy/module/hpy_universal/_vendored/test/test_unicodeobject.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/_vendored/test/test_unicodeobject.py
@@ -0,0 +1,64 @@
+# -*- encoding: utf-8 -*-
+
+from .support import HPyTest
+
+class TestUnicodeObject(HPyTest):
+
+    def test_Check(self):
+        mod = self.make_module("""
+            HPy_DEF_METH_O(f)
+            static HPy f_impl(HPyContext ctx, HPy self, HPy arg)
+            {
+                if (HPyUnicode_Check(ctx, arg))
+                    return HPy_Dup(ctx, ctx->h_True);
+                return HPy_Dup(ctx, ctx->h_False);
+            }
+            @EXPORT f HPy_METH_O
+            @INIT
+        """)
+        assert mod.f('hello') is True
+        assert mod.f(b'hello') is False
+
+    def test_FromString(self):
+        mod = self.make_module("""
+            HPy_DEF_METH_NOARGS(f)
+            static HPy f_impl(HPyContext ctx, HPy self)
+            {
+                return HPyUnicode_FromString(ctx, "foobar");
+            }
+            @EXPORT f HPy_METH_NOARGS
+            @INIT
+        """)
+        assert mod.f() == "foobar"
+
+    def test_FromWideChar(self):
+        mod = self.make_module("""
+            HPy_DEF_METH_O(f)
+            static HPy f_impl(HPyContext ctx, HPy self, HPy arg)
+            {
+                const wchar_t *buf = L"hell\xf2 world";
+                long n = HPyLong_AsLong(ctx, arg);
+                return HPyUnicode_FromWideChar(ctx, buf, n);
+            }
+            @EXPORT f HPy_METH_O
+            @INIT
+        """)
+        assert mod.f(-1) == "hell&#242; world"
+        assert mod.f(11) == "hell&#242; world"
+        assert mod.f(5) == "hell&#242;"
+
+
+    def test_AsUTF8String(self):
+        mod = self.make_module("""
+            HPy_DEF_METH_O(f)
+            static HPy f_impl(HPyContext ctx, HPy self, HPy arg)
+            {
+                return HPyUnicode_AsUTF8String(ctx, arg);
+            }
+            @EXPORT f HPy_METH_O
+            @INIT
+        """)
+        s = 'hell&#242;'
+        b = mod.f(s)
+        assert type(b) is bytes
+        assert b == s.encode('utf-8')

From pypy.commits at gmail.com  Thu Nov 28 06:17:37 2019
From: pypy.commits at gmail.com (antocuni)
Date: Thu, 28 Nov 2019 03:17:37 -0800 (PST)
Subject: [pypy-commit] pypy hpy: update the support machinery to take the
 new argument extra_templates;
 add test files for all the new hpy tests. Update the type declaration of
 HPyContext_s
Message-ID: <5ddfacd1.1c69fb81.aef9e.e9e8@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98166:cf17a3d5456f
Date: 2019-11-28 01:45 +0100
http://bitbucket.org/pypy/pypy/changeset/cf17a3d5456f/

Log:	update the support machinery to take the new argument
	extra_templates; add test files for all the new hpy tests. Update
	the type declaration of HPyContext_s

diff --git a/pypy/module/hpy_universal/llapi.py b/pypy/module/hpy_universal/llapi.py
--- a/pypy/module/hpy_universal/llapi.py
+++ b/pypy/module/hpy_universal/llapi.py
@@ -56,18 +56,34 @@
     struct _HPy_s h_True;
     struct _HPy_s h_False;
     struct _HPy_s h_ValueError;
-    void *ctx_Module_Create;
-    void *ctx_Dup;
-    void *ctx_Close;
-    void *ctx_Long_FromLong;
-    void *ctx_Long_AsLong;
-    void *ctx_Arg_Parse;
-    void *ctx_Number_Add;
-    void *ctx_Unicode_FromString;
-    void *ctx_Err_SetString;
-    void *ctx_FromPyObject;
-    void *ctx_AsPyObject;
-    void *ctx_CallRealFunctionFromTrampoline;
+    struct _HPy_s h_TypeError;
+    void * ctx_Module_Create;
+    void * ctx_Dup;
+    void * ctx_Close;
+    void * ctx_Long_FromLong;
+    void * ctx_Long_FromLongLong;
+    void * ctx_Long_FromUnsignedLongLong;
+    void * ctx_Long_AsLong;
+    void * ctx_Float_FromDouble;
+    void * ctx_Arg_Parse;
+    void * ctx_Number_Add;
+    void * ctx_Err_SetString;
+    void * ctx_Bytes_Check;
+    void * ctx_Bytes_Size;
+    void * ctx_Bytes_GET_SIZE;
+    void * ctx_Bytes_AsString;
+    void * ctx_Bytes_AS_STRING;
+    void * ctx_Unicode_FromString;
+    void * ctx_Unicode_Check;
+    void * ctx_Unicode_AsUTF8String;
+    void * ctx_Unicode_FromWideChar;
+    void * ctx_List_New;
+    void * ctx_List_Append;
+    void * ctx_Dict_New;
+    void * ctx_Dict_SetItem;
+    void * ctx_FromPyObject;
+    void * ctx_AsPyObject;
+    void * ctx_CallRealFunctionFromTrampoline;
 } _struct_HPyContext_s;
 typedef struct _HPyContext_s *HPyContext;
 
diff --git a/pypy/module/hpy_universal/test/support.py b/pypy/module/hpy_universal/test/support.py
--- a/pypy/module/hpy_universal/test/support.py
+++ b/pypy/module/hpy_universal/test/support.py
@@ -1,7 +1,7 @@
 import py
 import pytest
 from rpython.tool.udir import udir
-from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.gateway import interp2app, unwrap_spec, W_Root
 from pypy.module.hpy_universal.llapi import INCLUDE_DIR
 from pypy.module.hpy_universal._vendored.test import support as _support
 
@@ -26,9 +26,14 @@
                                                  keep=0)  # keep everything
         compiler = _support.ExtensionCompiler(tmpdir, 'universal', INCLUDE_DIR)
 
-        @unwrap_spec(source_template='text', name='text')
-        def descr_make_module(space, source_template, name='mytest'):
-            so_filename = compiler.compile_module(source_template, name)
+        @unwrap_spec(source_template='text', name='text', w_extra_templates=W_Root)
+        def descr_make_module(space, source_template, name='mytest',
+                              w_extra_templates=None):
+            if w_extra_templates is None:
+                extra_templates = ()
+            else:
+                import pdb;pdb.set_trace()
+            so_filename = compiler.compile_module(source_template, name, extra_templates)
             w_mod = space.appexec([space.newtext(so_filename), space.newtext(name)],
                 """(path, modname):
                     import hpy_universal
diff --git a/pypy/module/hpy_universal/test/test_bytesobject.py b/pypy/module/hpy_universal/test/test_bytesobject.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/test/test_bytesobject.py
@@ -0,0 +1,6 @@
+import pytest
+from pypy.module.hpy_universal._vendored.test.test_bytesobject import TestBytesObject as _TestBytesobject
+from .support import HPyAppTest
+
+class AppTestBytesObject(HPyAppTest, _TestBytesObject):
+    spaceconfig = {'usemodules': ['hpy_universal']}
diff --git a/pypy/module/hpy_universal/test/test_cpy_compat.py b/pypy/module/hpy_universal/test/test_cpy_compat.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/test/test_cpy_compat.py
@@ -0,0 +1,9 @@
+import pytest
+from pypy.module.hpy_universal._vendored.test.test_cpy_compat import TestCPythonCompatibility as _TestCPythonCompatibility
+from .support import HPyAppTest
+
+class AppTestUnicodeObject(HPyAppTest, _TestCPythonCompatibility):
+    spaceconfig = {'usemodules': ['hpy_universal']}
+
+    def setup_method(self, meth):
+        pytest.skip('IMPLEMENT ME')
diff --git a/pypy/module/hpy_universal/test/test_dictobject.py b/pypy/module/hpy_universal/test/test_dictobject.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/test/test_dictobject.py
@@ -0,0 +1,6 @@
+import pytest
+from pypy.module.hpy_universal._vendored.test.test_dictobject import TestDictObject as _TestDictobject
+from .support import HPyAppTest
+
+class AppTestDictObject(HPyAppTest, _TestDictObject):
+    spaceconfig = {'usemodules': ['hpy_universal']}
diff --git a/pypy/module/hpy_universal/test/test_listobject.py b/pypy/module/hpy_universal/test/test_listobject.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/test/test_listobject.py
@@ -0,0 +1,6 @@
+import pytest
+from pypy.module.hpy_universal._vendored.test.test_listobject import TestListObject as _TestListobject
+from .support import HPyAppTest
+
+class AppTestListObject(HPyAppTest, _TestListObject):
+    spaceconfig = {'usemodules': ['hpy_universal']}
diff --git a/pypy/module/hpy_universal/test/test_unicodeobject.py b/pypy/module/hpy_universal/test/test_unicodeobject.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/test/test_unicodeobject.py
@@ -0,0 +1,6 @@
+import pytest
+from pypy.module.hpy_universal._vendored.test.test_unicodeobject import TestUnicodeObject as _TestUnicodeobject
+from .support import HPyAppTest
+
+class AppTestUnicodeObject(HPyAppTest, _TestUnicodeObject):
+    spaceconfig = {'usemodules': ['hpy_universal']}

From pypy.commits at gmail.com  Thu Nov 28 06:17:39 2019
From: pypy.commits at gmail.com (antocuni)
Date: Thu, 28 Nov 2019 03:17:39 -0800 (PST)
Subject: [pypy-commit] pypy hpy: use the updated values for HPy_METH_*
Message-ID: <5ddfacd3.1c69fb81.5afa7.ee2d@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98167:e2ab328b16a8
Date: 2019-11-28 10:52 +0100
http://bitbucket.org/pypy/pypy/changeset/e2ab328b16a8/

Log:	use the updated values for HPy_METH_*

diff --git a/pypy/module/hpy_universal/interp_extfunc.py b/pypy/module/hpy_universal/interp_extfunc.py
--- a/pypy/module/hpy_universal/interp_extfunc.py
+++ b/pypy/module/hpy_universal/interp_extfunc.py
@@ -78,27 +78,27 @@
         flags = self.flags
         length = len(__args__.arguments_w)
 
-        if flags & llapi.METH_KEYWORDS:
+        if flags == llapi.HPy_METH_KEYWORDS:
             return self.call_keywords(space, __args__)
 
         if __args__.keywords:
             raise oefmt(space.w_TypeError,
                         "%s() takes no keyword arguments", self.name)
 
-        if flags & llapi.METH_NOARGS:
+        if flags == llapi.HPy_METH_NOARGS:
             if length == 0:
                 return self.call_noargs(space)
             raise oefmt(space.w_TypeError,
                         "%s() takes no arguments", self.name)
 
-        if flags & llapi.METH_O:
+        if flags == llapi.HPy_METH_O:
             if length != 1:
                 raise oefmt(space.w_TypeError,
                             "%s() takes exactly one argument (%d given)",
                             self.name, length)
             return self.call_o(space, __args__.arguments_w[0])
 
-        if flags & llapi.METH_VARARGS:
+        if flags == llapi.HPy_METH_VARARGS:
             return self.call_varargs(space, __args__.arguments_w)
         else:  # shouldn't happen!
             raise oefmt(space.w_RuntimeError, "unknown calling convention")
diff --git a/pypy/module/hpy_universal/interp_hpy.py b/pypy/module/hpy_universal/interp_hpy.py
--- a/pypy/module/hpy_universal/interp_hpy.py
+++ b/pypy/module/hpy_universal/interp_hpy.py
@@ -7,7 +7,7 @@
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.error import raise_import_error
 from pypy.interpreter.module import Module
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, oefmt
 
 from pypy.module.hpy_universal import llapi, handles, interp_extfunc
 from pypy.module.hpy_universal.state import State
@@ -40,6 +40,10 @@
         p = hpydef.c_m_methods
         i = 0
         while p[i].c_ml_name:
+            if not p[i].c_ml_flags & llapi._HPy_METH:
+                # we need to add support for legacy methods through cpyext
+                raise oefmt(space.w_NotImplementedError, "non-hpy method: %s",
+                            rffi.charp2str(p[i].c_ml_name))
             w_extfunc = interp_extfunc.W_ExtensionFunction(p[i], w_mod)
             space.setattr(w_mod, space.newtext(w_extfunc.name), w_extfunc)
             i += 1
diff --git a/pypy/module/hpy_universal/llapi.py b/pypy/module/hpy_universal/llapi.py
--- a/pypy/module/hpy_universal/llapi.py
+++ b/pypy/module/hpy_universal/llapi.py
@@ -136,13 +136,11 @@
 # HPyModule_Create
 HPyModuleDef._flds['c_m_methods'] = rffi.CArrayPtr(HPyMethodDef)
 
-METH_VARARGS  = 0x0001
-METH_KEYWORDS = 0x0002
-METH_NOARGS   = 0x0004
-METH_O        = 0x0008
-
-
-
+_HPy_METH = 0x100000
+HPy_METH_VARARGS  = 0x0001 | _HPy_METH
+HPy_METH_KEYWORDS = 0x0002 | _HPy_METH
+HPy_METH_NOARGS   = 0x0004 | _HPy_METH
+HPy_METH_O        = 0x0008 | _HPy_METH
 
 # ----------------------------------------------------------------
 
diff --git a/pypy/module/hpy_universal/test/support.py b/pypy/module/hpy_universal/test/support.py
--- a/pypy/module/hpy_universal/test/support.py
+++ b/pypy/module/hpy_universal/test/support.py
@@ -5,6 +5,7 @@
 from pypy.module.hpy_universal.llapi import INCLUDE_DIR
 from pypy.module.hpy_universal._vendored.test import support as _support
 
+COMPILER_VERBOSE = False
 
 class HPyAppTest(object):
 
@@ -24,7 +25,8 @@
         tmpdir = py.path.local.make_numbered_dir(rootdir=udir,
                                                  prefix=meth.__name__ + '-',
                                                  keep=0)  # keep everything
-        compiler = _support.ExtensionCompiler(tmpdir, 'universal', INCLUDE_DIR)
+        compiler = _support.ExtensionCompiler(tmpdir, 'universal', INCLUDE_DIR,
+                                              compiler_verbose=COMPILER_VERBOSE)
 
         @unwrap_spec(source_template='text', name='text', w_extra_templates=W_Root)
         def descr_make_module(space, source_template, name='mytest',

From pypy.commits at gmail.com  Thu Nov 28 06:17:41 2019
From: pypy.commits at gmail.com (antocuni)
Date: Thu, 28 Nov 2019 03:17:41 -0800 (PST)
Subject: [pypy-commit] pypy hpy: add one more builtin singleton
Message-ID: <5ddfacd5.1c69fb81.38e59.2474@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98168:d8aab0b1b105
Date: 2019-11-28 10:54 +0100
http://bitbucket.org/pypy/pypy/changeset/d8aab0b1b105/

Log:	add one more builtin singleton

diff --git a/pypy/module/hpy_universal/handles.py b/pypy/module/hpy_universal/handles.py
--- a/pypy/module/hpy_universal/handles.py
+++ b/pypy/module/hpy_universal/handles.py
@@ -5,6 +5,7 @@
     ('False', lambda space: space.w_False),
     ('True', lambda space: space.w_True),
     ('ValueError', lambda space: space.w_ValueError),
+    ('TypeError', lambda space: space.w_TypeError),
     ]
 
 

From pypy.commits at gmail.com  Thu Nov 28 06:17:43 2019
From: pypy.commits at gmail.com (antocuni)
Date: Thu, 28 Nov 2019 03:17:43 -0800 (PST)
Subject: [pypy-commit] pypy hpy: update to pyhandle/hpy 2aec7ae
Message-ID: <5ddfacd7.1c69fb81.cd084.cfd0@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98169:88722810d26d
Date: 2019-11-28 11:10 +0100
http://bitbucket.org/pypy/pypy/changeset/88722810d26d/

Log:	update to pyhandle/hpy 2aec7ae

diff --git a/pypy/module/hpy_universal/_vendored/test/support.py b/pypy/module/hpy_universal/_vendored/test/support.py
--- a/pypy/module/hpy_universal/_vendored/test/support.py
+++ b/pypy/module/hpy_universal/_vendored/test/support.py
@@ -79,7 +79,7 @@
         source = expand_template(template, name)
         filename = self.tmpdir.join(name + '.c')
         filename.write(source)
-        return filename
+        return str(filename)
 
     def compile_module(self, main_template, name, extra_templates):
         """

From pypy.commits at gmail.com  Thu Nov 28 06:17:44 2019
From: pypy.commits at gmail.com (antocuni)
Date: Thu, 28 Nov 2019 03:17:44 -0800 (PST)
Subject: [pypy-commit] pypy hpy: implement make_module(...,
 extra_sources=...)
Message-ID: <5ddfacd8.1c69fb81.3ede3.0a4f@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98170:95a38a6798b3
Date: 2019-11-28 11:10 +0100
http://bitbucket.org/pypy/pypy/changeset/95a38a6798b3/

Log:	implement make_module(..., extra_sources=...)

diff --git a/pypy/module/hpy_universal/test/support.py b/pypy/module/hpy_universal/test/support.py
--- a/pypy/module/hpy_universal/test/support.py
+++ b/pypy/module/hpy_universal/test/support.py
@@ -34,7 +34,8 @@
             if w_extra_templates is None:
                 extra_templates = ()
             else:
-                import pdb;pdb.set_trace()
+                items_w = space.unpackiterable(w_extra_templates)
+                extra_templates = [space.text_w(item) for item in items_w]
             so_filename = compiler.compile_module(source_template, name, extra_templates)
             w_mod = space.appexec([space.newtext(so_filename), space.newtext(name)],
                 """(path, modname):

From pypy.commits at gmail.com  Thu Nov 28 06:17:46 2019
From: pypy.commits at gmail.com (antocuni)
Date: Thu, 28 Nov 2019 03:17:46 -0800 (PST)
Subject: [pypy-commit] pypy hpy: finally! Collect all the api functions
 automatically and stick them into the context. We need to make this RPython,
 though
Message-ID: <5ddfacda.1c69fb81.c0ed5.887d@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98171:64a75c77d1cc
Date: 2019-11-28 11:52 +0100
http://bitbucket.org/pypy/pypy/changeset/64a75c77d1cc/

Log:	finally! Collect all the api functions automatically and stick them
	into the context. We need to make this RPython, though

diff --git a/pypy/module/hpy_universal/apiset.py b/pypy/module/hpy_universal/apiset.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/apiset.py
@@ -0,0 +1,39 @@
+import re
+from rpython.rtyper.annlowlevel import llhelper
+from rpython.rtyper.lltypesystem import lltype
+from rpython.rlib.objectmodel import specialize, llhelper_can_raise
+
+
+class APISet(object):
+
+    _PREFIX = re.compile(r'^_?HPy_?')
+
+    def __init__(self):
+        self.all_functions = []
+
+    def func(self, argtypes, restype):
+        def decorate(fn):
+            # attach various helpers to fn, so you can access things like
+            # HPyNumber_Add.get_llhelper(), HPyNumber_Add.basename, etc.
+
+            # get_llhelper
+            ll_functype = lltype.Ptr(lltype.FuncType(argtypes, restype))
+            @specialize.memo()
+            def make_wrapper(space):
+                @llhelper_can_raise
+                def wrapper(*args):
+                    return fn(space, *args)
+                return wrapper
+            def get_llhelper(space):
+                return llhelper(ll_functype, make_wrapper(space))
+            fn.get_llhelper = get_llhelper
+
+            # basename
+            fn.basename = self._PREFIX.sub(r'', fn.__name__)
+
+            # record it into the API
+            self.all_functions.append(fn)
+            return fn
+        return decorate
+
+API = APISet()
diff --git a/pypy/module/hpy_universal/interp_hpy.py b/pypy/module/hpy_universal/interp_hpy.py
--- a/pypy/module/hpy_universal/interp_hpy.py
+++ b/pypy/module/hpy_universal/interp_hpy.py
@@ -1,7 +1,5 @@
-from rpython.rtyper.annlowlevel import llhelper
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib.rdynload import dlopen, dlsym, DLOpenError
-from rpython.rlib.objectmodel import specialize, llhelper_can_raise
 from rpython.rlib import rutf8
 
 from pypy.interpreter.gateway import unwrap_spec
@@ -11,26 +9,11 @@
 
 from pypy.module.hpy_universal import llapi, handles, interp_extfunc
 from pypy.module.hpy_universal.state import State
+from pypy.module.hpy_universal.apiset import API
 from pypy.module.cpyext.api import generic_cpy_call_dont_convert_result
 
 
-def apifunc(argtypes, restype):
-    def decorate(fn):
-        ll_functype = lltype.Ptr(lltype.FuncType(argtypes, restype))
-        @specialize.memo()
-        def make_wrapper(space):
-            @llhelper_can_raise
-            def wrapper(*args):
-                return fn(space, *args)
-            return wrapper
-        def get_llhelper(space):
-            return llhelper(ll_functype, make_wrapper(space))
-        fn.get_llhelper = get_llhelper
-        return fn
-    return decorate
-
-
- at apifunc([llapi.HPyContext, lltype.Ptr(llapi.HPyModuleDef)], llapi.HPy)
+ at API.func([llapi.HPyContext, lltype.Ptr(llapi.HPyModuleDef)], llapi.HPy)
 def HPyModule_Create(space, ctx, hpydef):
     modname = rffi.constcharp2str(hpydef.c_m_name)
     w_mod = Module(space, space.newtext(modname))
@@ -51,20 +34,20 @@
     return handles.new(space, w_mod)
 
 
- at apifunc([llapi.HPyContext, llapi.HPy], llapi.HPy)
+ at API.func([llapi.HPyContext, llapi.HPy], llapi.HPy)
 def HPy_Dup(space, ctx, h):
     return handles.dup(space, h)
 
- at apifunc([llapi.HPyContext, llapi.HPy], lltype.Void)
+ at API.func([llapi.HPyContext, llapi.HPy], lltype.Void)
 def HPy_Close(space, ctx, h):
     handles.close(space, h)
 
- at apifunc([llapi.HPyContext, rffi.LONG], llapi.HPy)
+ at API.func([llapi.HPyContext, rffi.LONG], llapi.HPy)
 def HPyLong_FromLong(space, ctx, value):
     w_obj = space.newint(rffi.cast(lltype.Signed, value))
     return handles.new(space, w_obj)
 
- at apifunc([llapi.HPyContext, llapi.HPy], rffi.LONG)
+ at API.func([llapi.HPyContext, llapi.HPy], rffi.LONG)
 def HPyLong_AsLong(space, ctx, h):
     w_obj = handles.deref(space, h)
     #w_obj = space.int(w_obj)     --- XXX write a test for this
@@ -74,14 +57,14 @@
     #    ...
     return result
 
- at apifunc([llapi.HPyContext, llapi.HPy, llapi.HPy], llapi.HPy)
+ at API.func([llapi.HPyContext, llapi.HPy, llapi.HPy], llapi.HPy)
 def HPyNumber_Add(space, ctx, h1, h2):
     w_obj1 = handles.deref(space, h1)
     w_obj2 = handles.deref(space, h2)
     w_result = space.add(w_obj1, w_obj2)
     return handles.new(space, w_result)
 
- at apifunc([llapi.HPyContext, rffi.CCHARP], llapi.HPy)
+ at API.func([llapi.HPyContext, rffi.CCHARP], llapi.HPy)
 def HPyUnicode_FromString(space, ctx, utf8):
     w_obj = _maybe_utf8_to_w(space, utf8)
     return handles.new(space, w_obj)
@@ -95,7 +78,7 @@
         raise   # XXX do something
     return space.newtext(s, length)
 
- at apifunc([llapi.HPyContext, llapi.HPy, rffi.CCHARP], lltype.Void)
+ at API.func([llapi.HPyContext, llapi.HPy, rffi.CCHARP], lltype.Void)
 def HPyErr_SetString(space, ctx, h_exc_type, utf8):
    w_obj = _maybe_utf8_to_w(space, utf8)
    w_exc_type = handles.deref(space, h_exc_type)
diff --git a/pypy/module/hpy_universal/state.py b/pypy/module/hpy_universal/state.py
--- a/pypy/module/hpy_universal/state.py
+++ b/pypy/module/hpy_universal/state.py
@@ -5,6 +5,7 @@
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib.objectmodel import specialize
 from pypy.module.hpy_universal import llapi, handles
+from pypy.module.hpy_universal.apiset import API
 
 CONTEXT_FIELDS = unrolling_iterable(llapi.HPyContext.TO._names)
 CONSTANT_NAMES = unrolling_iterable([name for name, _ in handles.CONSTANTS])
@@ -50,31 +51,11 @@
                 h_struct.c__i = i
             i = i + 1
 
-        # XXX collect all these functions automatically
-        from pypy.module.hpy_universal import interp_hpy
-        
-        funcptr = interp_hpy.HPyModule_Create.get_llhelper(space)
-        self.ctx.c_ctx_Module_Create = rffi.cast(rffi.VOIDP, funcptr)
-        #
-        funcptr = interp_hpy.HPy_Dup.get_llhelper(space)
-        self.ctx.c_ctx_Dup = rffi.cast(rffi.VOIDP, funcptr)
-        #
-        funcptr = interp_hpy.HPy_Close.get_llhelper(space)
-        self.ctx.c_ctx_Close = rffi.cast(rffi.VOIDP, funcptr)
-        #
-        funcptr = interp_hpy.HPyLong_FromLong.get_llhelper(space)
-        self.ctx.c_ctx_Long_FromLong = rffi.cast(rffi.VOIDP, funcptr)
-        #
-        funcptr = interp_hpy.HPyLong_AsLong.get_llhelper(space)
-        self.ctx.c_ctx_Long_AsLong = rffi.cast(rffi.VOIDP, funcptr)
-        #
-        funcptr = interp_hpy.HPyNumber_Add.get_llhelper(space)
-        self.ctx.c_ctx_Number_Add = rffi.cast(rffi.VOIDP, funcptr)
-        #
-        funcptr = interp_hpy.HPyUnicode_FromString.get_llhelper(space)
-        self.ctx.c_ctx_Unicode_FromString = rffi.cast(rffi.VOIDP, funcptr)
-        #
-        funcptr = interp_hpy.HPyErr_SetString.get_llhelper(space)
-        self.ctx.c_ctx_Err_SetString = rffi.cast(rffi.VOIDP, funcptr)
-        #
+        # XXX this is not RPython, we need a way to turn this into an
+        # unrolling_iterable
+        for func in API.all_functions:
+            funcptr = rffi.cast(rffi.VOIDP, func.get_llhelper(space))
+            ctx_field = 'c_ctx_' + func.basename
+            setattr(self.ctx, ctx_field, funcptr)
+
         self.ctx.c_ctx_Arg_Parse = rffi.cast(rffi.VOIDP, llapi.DONT_CALL_ctx_Arg_Parse)
diff --git a/pypy/module/hpy_universal/test/test_apiset.py b/pypy/module/hpy_universal/test/test_apiset.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/hpy_universal/test/test_apiset.py
@@ -0,0 +1,36 @@
+from rpython.rtyper.lltypesystem import lltype
+from pypy.module.hpy_universal.api import APISet
+
+class TestAPISet(object):
+
+    def test_func(self):
+        api = APISet()
+        @api.func([lltype.Signed, lltype.Signed], lltype.Float)
+        def divide(space, a, b):
+            return float(a)/b
+        #
+        assert divide(None, 5, 2) == 2.5
+        assert api.all_functions == [divide]
+
+    def test_basename(self):
+        api = APISet()
+        @api.func([], lltype.Void)
+        def HPyFoo_Bar(space):
+            return None
+        @api.func([], lltype.Void)
+        def _HPyFoo_Internal(space):
+            return None
+
+        assert HPyFoo_Bar.basename == 'Foo_Bar'
+        assert _HPyFoo_Internal.basename == 'Foo_Internal'
+
+    def test_llhelper(self):
+        api = APISet()
+        @api.func([lltype.Signed, lltype.Signed], lltype.Float)
+        def divide(space, a, b):
+            assert space == 'MySpace'
+            return float(a)/b
+        #
+        space = 'MySpace'
+        lldivide = divide.get_llhelper(space)
+        assert lldivide(5, 2) == 2.5

From pypy.commits at gmail.com  Thu Nov 28 06:17:47 2019
From: pypy.commits at gmail.com (antocuni)
Date: Thu, 28 Nov 2019 03:17:47 -0800 (PST)
Subject: [pypy-commit] pypy hpy: fix translation for the part which was
 broken by 64a75c77d1cc. Translation is still broken because of other
 problems though, will be fixed later
Message-ID: <5ddfacdb.1c69fb81.b4e6f.d7cc@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: hpy
Changeset: r98172:961b3a276537
Date: 2019-11-28 12:16 +0100
http://bitbucket.org/pypy/pypy/changeset/961b3a276537/

Log:	fix translation for the part which was broken by 64a75c77d1cc.
	Translation is still broken because of other problems though, will
	be fixed later

diff --git a/pypy/module/hpy_universal/apiset.py b/pypy/module/hpy_universal/apiset.py
--- a/pypy/module/hpy_universal/apiset.py
+++ b/pypy/module/hpy_universal/apiset.py
@@ -1,6 +1,7 @@
 import re
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rtyper.lltypesystem import lltype
+from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib.objectmodel import specialize, llhelper_can_raise
 
 
@@ -11,6 +12,10 @@
     def __init__(self):
         self.all_functions = []
 
+    def _freeze_(self):
+        self.all_functions = unrolling_iterable(self.all_functions)
+        return True
+
     def func(self, argtypes, restype):
         def decorate(fn):
             # attach various helpers to fn, so you can access things like
diff --git a/pypy/module/hpy_universal/interp_hpy.py b/pypy/module/hpy_universal/interp_hpy.py
--- a/pypy/module/hpy_universal/interp_hpy.py
+++ b/pypy/module/hpy_universal/interp_hpy.py
@@ -26,7 +26,7 @@
             if not p[i].c_ml_flags & llapi._HPy_METH:
                 # we need to add support for legacy methods through cpyext
                 raise oefmt(space.w_NotImplementedError, "non-hpy method: %s",
-                            rffi.charp2str(p[i].c_ml_name))
+                            rffi.constcharp2str(p[i].c_ml_name))
             w_extfunc = interp_extfunc.W_ExtensionFunction(p[i], w_mod)
             space.setattr(w_mod, space.newtext(w_extfunc.name), w_extfunc)
             i += 1

From pypy.commits at gmail.com  Fri Nov 29 04:34:01 2019
From: pypy.commits at gmail.com (mattip)
Date: Fri, 29 Nov 2019 01:34:01 -0800 (PST)
Subject: [pypy-commit] pypy default: tweak build documentation
Message-ID: <5de0e609.1c69fb81.45d66.1cce@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r98176:1b7eb71c73ea
Date: 2019-11-29 10:08 +0200
http://bitbucket.org/pypy/pypy/changeset/1b7eb71c73ea/

Log:	tweak build documentation

diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -60,12 +60,9 @@
 Install build-time dependencies
 -------------------------------
 (**Note**: for some hints on how to translate the Python interpreter under
-Windows, see the `windows document`_ . For hints on how to cross-compile in
-a chroot using scratchbox2, see the `arm document`_ in the
-`RPython documentation`_)
+Windows, see the `windows document`_ . 
 
 .. _`windows document`: windows.html
-.. _`arm document`: http://rpython.readthedocs.org/en/latest/arm.html
 .. _`RPython documentation`: http://rpython.readthedocs.org
 
 The host Python needs to have CFFI installed. If translating on PyPy, CFFI is
@@ -138,7 +135,7 @@
 
     dnf install gcc make libffi-devel pkgconfig zlib-devel bzip2-devel \
     sqlite-devel ncurses-devel expat-devel openssl-devel tk-devel \
-    gdbm-devel python-cffi\
+    gdbm-devel python-cffi gc-devel\
     xz-devel  # For lzma on PyPy3.
 
 On SLES11::
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -155,7 +155,7 @@
 the `get_externals.py` utility to checkout the proper branch for your platform
 and PyPy version.
 
-.. _subrepository:  https://bitbucket.org/pypy/external
+.. _subrepository:  https://bitbucket.org/pypy/externals
 
 Using the mingw compiler
 ------------------------

From pypy.commits at gmail.com  Fri Nov 29 04:34:03 2019
From: pypy.commits at gmail.com (mattip)
Date: Fri, 29 Nov 2019 01:34:03 -0800 (PST)
Subject: [pypy-commit] pypy py3.6: merge default into branch
Message-ID: <5de0e60b.1c69fb81.cc12f.55fd@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r98177:399112d66ce0
Date: 2019-11-29 11:17 +0200
http://bitbucket.org/pypy/pypy/changeset/399112d66ce0/

Log:	merge default into branch

diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -60,12 +60,9 @@
 Install build-time dependencies
 -------------------------------
 (**Note**: for some hints on how to translate the Python interpreter under
-Windows, see the `windows document`_ . For hints on how to cross-compile in
-a chroot using scratchbox2, see the `arm document`_ in the
-`RPython documentation`_)
+Windows, see the `windows document`_ . 
 
 .. _`windows document`: windows.html
-.. _`arm document`: http://rpython.readthedocs.org/en/latest/arm.html
 .. _`RPython documentation`: http://rpython.readthedocs.org
 
 The host Python needs to have CFFI installed. If translating on PyPy, CFFI is
@@ -138,7 +135,7 @@
 
     dnf install gcc make libffi-devel pkgconfig zlib-devel bzip2-devel \
     sqlite-devel ncurses-devel expat-devel openssl-devel tk-devel \
-    gdbm-devel python-cffi\
+    gdbm-devel python-cffi gc-devel\
     xz-devel  # For lzma on PyPy3.
 
 On SLES11::
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -155,7 +155,7 @@
 the `get_externals.py` utility to checkout the proper branch for your platform
 and PyPy version.
 
-.. _subrepository:  https://bitbucket.org/pypy/external
+.. _subrepository:  https://bitbucket.org/pypy/externals
 
 Using the mingw compiler
 ------------------------

From pypy.commits at gmail.com  Sat Nov 30 10:58:24 2019
From: pypy.commits at gmail.com (Yannick_Jadoul)
Date: Sat, 30 Nov 2019 07:58:24 -0800 (PST)
Subject: [pypy-commit] pypy default: merge backport-decode_timeval_ns-py3.7
 into default
Message-ID: <5de291a0.1c69fb81.d23aa.c0a7@mx.google.com>

Author: Yannick Jadoul <yannick.jadoul at belgacom.net>
Branch: 
Changeset: r98193:d3c8d438315b
Date: 2019-11-30 16:57 +0100
http://bitbucket.org/pypy/pypy/changeset/d3c8d438315b/

Log:	merge backport-decode_timeval_ns-py3.7 into default

diff --git a/rpython/rlib/rtime.py b/rpython/rlib/rtime.py
--- a/rpython/rlib/rtime.py
+++ b/rpython/rlib/rtime.py
@@ -9,7 +9,7 @@
 from rpython.rtyper.tool import rffi_platform
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib.objectmodel import register_replacement_for
-from rpython.rlib.rarithmetic import intmask, UINT_MAX
+from rpython.rlib.rarithmetic import intmask, r_int64, UINT_MAX
 from rpython.rlib import rposix
 
 _WIN32 = sys.platform.startswith('win')
@@ -94,6 +94,10 @@
     return (float(rffi.getintfield(t, 'c_tv_sec')) +
             float(rffi.getintfield(t, 'c_tv_usec')) * 0.000001)
 
+def decode_timeval_ns(t):
+    return (r_int64(rffi.getintfield(t, 'c_tv_sec')) * 10**9 +
+            r_int64(rffi.getintfield(t, 'c_tv_usec')) * 10**3)
+
 
 def external(name, args, result, compilation_info=eci, **kwds):
     return rffi.llexternal(name, args, result,

From pypy.commits at gmail.com  Sat Nov 30 13:09:27 2019
From: pypy.commits at gmail.com (mattip)
Date: Sat, 30 Nov 2019 10:09:27 -0800 (PST)
Subject: [pypy-commit] pypy default: fixes for CentOS 6 (manylinux image)
Message-ID: <5de2b057.1c69fb81.5aba3.fd65@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r98194:d5e7cbe1eb5f
Date: 2019-11-30 17:58 +0000
http://bitbucket.org/pypy/pypy/changeset/d5e7cbe1eb5f/

Log:	fixes for CentOS 6 (manylinux image)

diff --git a/pypy/module/_minimal_curses/fficurses.py b/pypy/module/_minimal_curses/fficurses.py
--- a/pypy/module/_minimal_curses/fficurses.py
+++ b/pypy/module/_minimal_curses/fficurses.py
@@ -7,8 +7,16 @@
 
 # We cannot trust ncurses5-config, it's broken in various ways in
 # various versions.  For example it might not list -ltinfo even though
-# it's needed, or --cflags might be completely empty.  On Ubuntu 10.04
-# it gives -I/usr/include/ncurses, which doesn't exist at all.  Crap.
+# it's needed, or --cflags might be completely empty. Crap.
+
+IS_CENTOS_6_10 = False
+try:
+    with open('/etc/redhat-release') as fid:
+        for line in fid:
+            if 'CentOS release 6.10' in line:
+                IS_CENTOS_6_10 = True
+except IOError:
+    pass
 
 def try_cflags():
     yield ExternalCompilationInfo(includes=['curses.h', 'term.h'])
@@ -20,8 +28,9 @@
                                             'ncurses/term.h'])
 
 def try_ldflags():
+    yield ExternalCompilationInfo(libraries=['curses', 'tinfo'])
     yield ExternalCompilationInfo(libraries=['curses'])
-    yield ExternalCompilationInfo(libraries=['curses', 'tinfo'])
+    yield ExternalCompilationInfo(libraries=['ncurses', 'tinfo'])
     yield ExternalCompilationInfo(libraries=['ncurses'])
     yield ExternalCompilationInfo(libraries=['ncurses'],
                                   library_dirs=['/usr/lib64'])
@@ -29,6 +38,8 @@
                                   library_dirs=['/usr/lib64'])
 
 def try_tools():
+    if IS_CENTOS_6_10:
+        return
     try:
         yield ExternalCompilationInfo.from_pkg_config("ncurses")
     except Exception:
diff --git a/pypy/module/zlib/test/test_zlib.py b/pypy/module/zlib/test/test_zlib.py
--- a/pypy/module/zlib/test/test_zlib.py
+++ b/pypy/module/zlib/test/test_zlib.py
@@ -345,7 +345,7 @@
 
         assert (d1 + from_copy) == (d1 + from_compressor)
 
-    @py.test.mark.skipif(rzlib.ZLIB_VERSION == '1.2.8', reason='does not error check')
+    @py.test.mark.skipif(rzlib.ZLIB_VERSION in ('1.2.8', '1.2.3'), reason='does not error check')
     def test_cannot_copy_compressor_with_stream_in_inconsistent_state(self):
         if self.runappdirect: skip("can't run with -A")
         compressor = self.zlib.compressobj()

From pypy.commits at gmail.com  Sat Nov 30 14:24:56 2019
From: pypy.commits at gmail.com (mattip)
Date: Sat, 30 Nov 2019 11:24:56 -0800 (PST)
Subject: [pypy-commit] buildbot default: build newer version of xz for lzma
 module
Message-ID: <5de2c208.1c69fb81.78e6.3ab9@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r1111:a27fd9b3c9f3
Date: 2019-11-30 20:23 +0100
http://bitbucket.org/pypy/buildbot/changeset/a27fd9b3c9f3/

Log:	build newer version of xz for lzma module

diff --git a/docker/Dockerfile b/docker/Dockerfile
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -24,17 +24,13 @@
 # docker run -it --rm -v<abspath/to/builder/dir>:/build_dir> \
 #     -ePYPY_MAKE_PORTABLE=1 buildslave_x86_64
 #
-# Use HOME since the default for docker is /, which
-# is not available to the buildslave_x86_64 user
-#
 # You might want to keep the PYPY_USESSION_DIR where the testing/building
 # artifacts are. Docker will not do this for you, so do something like this
 # to save the files outside the docker
 #
 # mkdir -p build_dir/tmp
 # docker run -it --rm -v<abspath/to/builder/dir>:/build_dir> \
-#     -ePYPY_USESSION_DIR=/build_dir/tmp -ePYPY_MAKE_PORTABLE=1 \
-#     -eHOME=/buildslave_home -v/home/buildslave:/buildslave_home buildslave_x86_64
+#     -ePYPY_USESSION_DIR=/build_dir/tmp -ePYPY_MAKE_PORTABLE=1 buildslave_x86_64
 #
 # To enter the buildslave image, add a shell command to the end
 #
@@ -58,6 +54,10 @@
 ADD install_libffi.sh /root/install_libffi.sh
 RUN sh install_libffi.sh manylinux2010 2>&1 | tee /root/install_libffi.log
 RUN sh install_openssl.sh manylinux2010 2>&1 | tee /root/install_openssl.log
+ADD install_xz5.sh /root/install_xz5.sh
+ADD lasse_collin_pubkey.txt /root/lasse_collin_pubkey.txt
+ADD xz-5.2.4.tar.gz.sig /root/xz-5.2.4.tar.gz.sig
+RUN sh install_xz5.sh manylinux2010 m32 # 2>&1 | tee /root/install_xz5.log
 
 # get a pypy for translation. On x86_64 we can use portable pypy
 RUN wget -q https://bitbucket.org/squeaky/portable-pypy/downloads/pypy-7.0.0-linux_x86_64-portable.tar.bz2 -O - | tar -C /opt -xj
@@ -70,7 +70,7 @@
 RUN virtualenv -p /opt/python/cp27-cp27mu/bin/python /python27_virt
 ENV PATH=/python27_virt/bin:$PATH
 RUN pip install --upgrade pip setuptools
-RUN pip install buildbot-slave pytest hypothesis cffi vmprof mercurial virtualenv
+RUN pip install buildbot-slave pytest hypothesis==4.42 cffi vmprof mercurial virtualenv
 
 # Define a user
 ARG BUILDSLAVE_UID=1001
diff --git a/docker/Dockerfile32 b/docker/Dockerfile32
--- a/docker/Dockerfile32
+++ b/docker/Dockerfile32
@@ -24,17 +24,13 @@
 # docker run -it --rm -v<abspath/to/builder/dir>:/build_dir> \
 #     -ePYPY_MAKE_PORTABLE=1 buildslave_i686
 #
-# Use HOME since the default for docker is /, which
-# is not available to the buildslave_i686 user
-#
 # You might want to keep the PYPY_USESSION_DIR where the testing/building
 # artifacts are. Docker will not do this for you, so do something like this
 # to save the files outside the docker
 #
 # mkdir -p build_dir/tmp
 # docker run -it --rm -v<abspath/to/builder/dir>:/build_dir> \
-#     -ePYPY_USESSION_DIR=/build_dir/tmp -ePYPY_MAKE_PORTABLE=1 \
-#     -eHOME=/buildslave_home -v/home/buildslave:/buildslave_home buildslave_i686
+#     -ePYPY_USESSION_DIR=/build_dir/tmp -ePYPY_MAKE_PORTABLE=1 buildslave_i686
 #
 # To enter the buildslave image, add a shell command to the end
 #
@@ -58,6 +54,10 @@
 ADD install_libffi.sh /root/install_libffi.sh
 RUN sh install_libffi.sh manylinux2010 m32 2>&1 | tee /root/install_libffi.log
 RUN sh install_openssl.sh manylinux2010 m32 2>&1 | tee /root/install_openssl.log
+ADD install_xz5.sh /root/install_xz5.sh
+ADD lasse_collin_pubkey.txt /root/lasse_collin_pubkey.txt
+ADD xz-5.2.4.tar.gz.sig /root/xz-5.2.4.tar.gz.sig
+RUN sh install_xz5.sh manylinux2010 m32 # 2>&1 | tee /root/install_xz5.log
 
 # get a pypy for translation. On x86_64 we can use portable pypy
 # on i686 we used python to bootstrap a translation, packaged and copied it out
@@ -76,7 +76,7 @@
 RUN virtualenv -p /opt/python/cp27-cp27mu/bin/python /python27_virt
 ENV PATH=/python27_virt/bin:$PATH
 RUN pip install --upgrade pip setuptools
-RUN pip install buildbot-slave pytest hypothesis cffi vmprof mercurial virtualenv
+RUN pip install buildbot-slave pytest hypothesis==4.42 cffi vmprof mercurial virtualenv
 
 # Define a user
 ARG BUILDSLAVE_UID=1001
diff --git a/docker/lasse_collin_pubkey.txt b/docker/lasse_collin_pubkey.txt
new file mode 100644
--- /dev/null
+++ b/docker/lasse_collin_pubkey.txt
@@ -0,0 +1,75 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQINBEzEOZIBEACxg/IuXERlDB48JBWmF4NxNUuuup1IhJAJyFGFSKh3OGAO2Ard
+sNuRLjANsFXA7m7P5eTFcG+BoHHuAVYmKnI3PPZtHVLnUt4pGItPczQZ2BE1WpcI
+ayjGTBJeKItX3Npqg9D/odO9WWS1i3FQPVdrLn0YH37/BA66jeMQCRo7g7GLpaNf
+IrvYGsqTbxCwsmA37rpE7oyU4Yrf74HT091WBsRIoq/MelhbxTDMR8eu/dUGZQVc
+Kj3lN55RepwWwUUKyqarY0zMt4HkFJ7v7yRL+Cvzy92Ouv4Wf2FlhNtEs5LE4Tax
+W0PO5AEmUoKjX87SezQK0f652018b4u6Ex52cY7p+n5TII/UyoowH6+tY8UHo9yb
+fStrqgNE/mY2bhA6+AwCaOUGsFzVVPTbjtxL3HacUP/jlA1h78V8VTvTs5d55iG7
+jSqR9o05wje8rwNiXXK0xtiJahyNzL97Kn/DgPSqPIi45G+8nxWSPFM5eunBKRl9
+vAnsvwrdPRsR6YR3uMHTuVhQX9/CY891MHkaZJ6wydWtKt3yQwJLYqwo5d4DwnUX
+CduUwSKv+6RmtWI5ZmTQYOcBRcZyGKml9X9Q8iSbm6cnpFXmLrNQwCJN+D3SiYGc
+MtbltZo0ysPMa6Xj5xFaYqWk/BI4iLb2Gs+ByGo/+a0Eq4XYBMOpitNniQARAQAB
+tCdMYXNzZSBDb2xsaW4gPGxhc3NlLmNvbGxpbkB0dWthYW5pLm9yZz6JAlEEEwEK
+ADsCGwMCHgECF4AECwkIBwMVCggFFgIDAQAWIQQ2kMJAzlG0Zw0wrRw47nV9aRhG
+IAUCXERzcAUJEx223gAKCRA47nV9aRhGIMgSEACivA8+mQnVbL6mubeAMZpMr5rR
+c7VanU9kC5IdC0RcMq/hcNfOJR5ncwM+JY+QCQVxAHBvHKAuHhC/r1Wl+WB6LYzD
+wbVszpNdfSnUci+WLETw0BCGhPAQHuUWCMYoQEnFSLxqfjshO1CCCfGHhwkhjplG
++A5eRZua74cZoe4k05OQCwY1mynDzyw1eyYuFZb+TW2YZ3a9/L1UJ7y/BgvxZtq1
+Zllbt4NhYLZc3V1uzRhtU63+M9gL29PE8XM+9jDMrSDrna+3SiwDEJP2SlbtNuxf
+W8xKtTkmdvwpbV7xBId8NvmABSQX2TfaEz9yZ1N6U6HzT95v1dsxM2us7ySCkyoQ
+KS9BEfpPpsdFL1VQvfsHdU580eCkHhFxPxYxyFfxwlmSeovUlN75e60kF1Lzqagn
+l6Jd3+pb/hGXd4PBz1Lrl9a1ERuZzKyMBG2ih6RRVBE7s/vrLF9+eRkjALjmTj/d
+cyOaZvvWhmbQRf7d8uZoLp3zkyKsWrk/y+u3J1FmshRN4X5tF66Q0ZME/RimidGA
+reiFZ3iTILNu3xd8oMIPENY/KVX7j/HZFGbOIpKAKqHiJEgX2vub5CVSBuXp/M0a
+17FJ9l8puy6hH0hlDLplRk4SbzvrALh2GK6pOpFJCm6xPto/wOdlL+l/lgwOHqRC
+DxfB4ErQGEn3qlp3jIkCOAQTAQIAIgUCTMQ5kgIbAwYLCQgHAwIGFQgCCQoLBBYC
+AwECHgECF4AACgkQOO51fWkYRiAg4A/7BXKwoRaXrMbMPOW7vuVF7c2IKB2Yqzn1
+vLBCwuEHkqY237lDcXY4/5LR+1gcZ3Duw1n/BRSm0FBdvyX/JTWiWNSDUkKAO/0l
+T2Tg44YLrDT3bzwu8dbU9xQt6kH+SCOHvv5Oe4k79l5mro6fF3H1M0bN63x/YoFY
+ojy09D7/JptY82oR4f/VdKnfZLJcCViCb0wp8SD2NkDAudKg+K+7PD8HlTWklQQg
+TZdRXxVZKIJeU42aJDqnRbAhJd64YHyClhqut9F5LUmiP5qfLfNhkKDhNOwk2Blr
+BGBJkSd7wPyzcX4Mun/L6YspHjbeVMt9TD7HQlo+OOd2OjAHCx6pqwkXnzeLPEaE
+cPdQ1SHgrBViAxX3DNPubLP0Knw8XwFu96EuhHZgexE1W7bB4LFsJyXAc5k1PqPD
+CLsAauxmvI2OfI7opG/8wyxDvNgoPjG8fZNAgY0REqPC0JnTXChH31IxUmhNotH8
+tD3DDTZOHw05n5MwwUrEE9xiETVDfFQcMLfxZ9KLz+BC2g1t5LYublRgnCMNJzFg
+sNUMM02CphABzl/LCLnumr0eyQQ/weV4twEhLwSDmqLYHL0EdYW0Y3CnnU9vmYxQ
+cXKbstS71sEJJYBBmSBbf9GxkOY8BRNtwVwY0kPgxv1WqdVBiAFvfB+pyAsrax9B
+3UeB7ZSwRD6JAhwEEAEKAAYFAlS25GwACgkQlbYYGy0z6ew92Q//ZA9/6piQtoW4
+PwP/1DtWGyKU8hwR+9FG669iPk/dAG+yoEJtFMOUpg/FUFmCX8Bc4oEHsCVyLxKt
+DcCVUIRcYNSFi5hTZaBEbwsOlDT37gtlfIIu34hhHRccKaLnN/N9gNMNw8wGh9xg
+Q/KtxZwcbk/bZIlDkKTJkFBRAekdEGAFDWb/AZOy+LQxS8ZAh1eWkfV0i8opmK9k
+gPXtLE0WSsqtYyGs58z+BFE9NH3tEUwK6jSvtuLwQl4UrICNbKthcpb8WwH6UXzb
+q3QNSYVOpf/cqRdBJA6bvb/ku/xyKVL08lGmxD9v1b137R7mafDAFPTsvH2Mt/0V
+YuhtWav3r1Bl9QksDxt2DTS8wiWDUBetGqOVdcw7vBrXPEWDNBmxeJXsiJ7zJlR+
+9wrJOm6RV2+l1IPxu96EaPS+kTNBijKrhxb67bww8BTEWTd0wcdJmgWRkM8SIstp
+IKqd0L2TFYph2/NtrBhRg+DIEPJPpSTGsUMcCEXCZPQ+cIdlQKsWpk0tZ62DlvEl
+r7E+wgUSQolRfx5KrpZifiS2zQlhzdXv28CJhsVbLyw5fUAWUKIH/dCo5NKsNLk2
+Lc5DH9VWnFgxAAtW290FqeK/4ulMq7Vs1dQSwyHM2Ni3QqqeaiOrh8gbSY5CMLFN
+Y3HYRwuTYPa3AobsozCzBj0Zdf/6AFe5Ag0ETMQ5kgEQAL/FwKdjxgPxtSpgq1SM
+zgZtTTyLqhgGD3NZfadHWHYRIL38NDV3JeTA79Y2zj2dj7KQPDT+0aqeizTV2E3j
+P3iCQ53VOT4consBaQAgKexpptnS+T1DobtICFJ0GGzf0HRj6KO2zSOuOitWPWlU
+wbvX7M0LLI2+hqlx0jTPqbJFZ/Za6KTtbS6xdCPVUpUqYZQpokEZcwQmUp8Q+lGo
+JD2sNYCZyap63X/aAOgCGr2RXYddOH5e8vGzGW+mwtCv+WQ9Ay35mGqI5MqkbZd1
+Qbuv2b1647E/QEEucfRHVbJVKGGPpFMUJtcItyyIt5jo+r9CCL4Cs47dF/9/RNwu
+NvpvHXUyqMBQdWNZRMx4k/NGD/WviPi9m6mIMui6rOQsSOaqYdcUX4Nq2Orr3Oaz
+2JPQdUfeI23iot1vK8hxvUCQTV3HfJghizN6spVl0yQOKBiE8miJRgrjHilH3hTb
+xoo42xDkNAq+CQo3QAm1ibDxKCDq0RcWPjcCRAN/Q5MmpcodpdKkzV0yGIS4g7s5
+frVrgV/kox2r4/Yxsr8K909+4H82AjTKGX/BmsQFCTAqBk6p7I0zxjIqJ/w33TZB
+Q0Pn4r3WIlUPafzY6a9/LAvN1fHRxf9SpCByJsszD03Qu5f5TB8gthsdnVmTo7jj
+iordEKMtw2aEMLzdWWTQ/TNVABEBAAGJAjwEGAEKACYCGwwWIQQ2kMJAzlG0Zw0w
+rRw47nV9aRhGIAUCXERzXAUJEx22ygAKCRA47nV9aRhGIDqVD/46sXUGfW5A2dP5
+vk9d0zTERwUAvgzZfZJWTJ38AERiqCbFLonVbqMF4Yj2rCat50nSVvI8UnHO61qT
+SWB/nwdCjTgmHl4N/hhplWSnY/+OcMOgHJ7MF3w7aBvCZqgVN6h/2w2oUCI18KHF
+/KkoWu66DrqWhOzWP0feI3UCgLuzZP7KJ6oE6yv3w0I8vV/2G4Mm7HSgstLur5vZ
+yO/MyiV/x2OR33H25HhwHEzZMm0vO+EAR4FWcLqX/70rv5Qy4QY0aLSC5EvY3X9Q
+4P0QxiEjmRsGgm7dh03Pxbr01JH5sIW6gnrCs0oxmdnLt8XyMYkvGdUdllVUe1XX
+0UT6buHetWNOv6RoS9g0E+GEI7I7qEl7x9z7rB3AWwOU6FFteggBFfXI/AmRIfBg
+/NUdM4Co1sIjyyyQcGgIYiq9MvyGRSey9/td9yaQpB02oITfyqwShRY3a2CnXr6l
+nW4uwa0LrNA6eBDVub0GLADvJiqwagt8uJqSBq8aGQgn9xhPUptKJlwKfKYHVdVS
+n95tAusFKQ9ECgW3Tteu76pmwBhgtieWqcW+fzI04+nDD2xSozlEaEoaDHD4Ti70
+wW3VWzUd2E6HDlWw+uG7Ll9E/O7fCsZ2obEIUWRjzQKb1992CcfUb/kuwF2CtAVV
+aGKSZLbWRS47D8RFJS+CAn6a3TqNLQ==
+=TZ8V
+-----END PGP PUBLIC KEY BLOCK-----
diff --git a/docker/xz-5.2.4.tar.gz.sig b/docker/xz-5.2.4.tar.gz.sig
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..639691636ff9921926d1e2629a312ecb6c05529d
GIT binary patch

[cut]


From pypy.commits at gmail.com  Sat Nov 30 14:24:58 2019
From: pypy.commits at gmail.com (mattip)
Date: Sat, 30 Nov 2019 11:24:58 -0800 (PST)
Subject: [pypy-commit] buildbot default: add missing file
Message-ID: <5de2c20a.1c69fb81.cd34e.d175@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r1112:efd944b812ac
Date: 2019-11-30 20:24 +0100
http://bitbucket.org/pypy/buildbot/changeset/efd944b812ac/

Log:	add missing file

diff --git a/docker/install_xz5.sh b/docker/install_xz5.sh
new file mode 100644
--- /dev/null
+++ b/docker/install_xz5.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+set -xe
+XZ5_FILE=xz-5.2.4.tar.gz
+
+wget -q "https://tukaani.org/xz/${XZ5_FILE}"
+gpg --import lasse_collin_pubkey.txt
+gpg --verify ${XZ5_FILE}.sig ${XZ5_FILE}
+
+tar zxf ${XZ5_FILE}
+pushd xz-5*
+if [ "$2" == "m32" ]; then
+  setarch i386 ./configure --prefix=/usr/local CFLAGS="-m32"
+else
+  ./configure --prefix=/usr/local
+fi
+make install
+popd
+rm -rf xz-5*

From pypy.commits at gmail.com  Sat Nov 30 14:30:39 2019
From: pypy.commits at gmail.com (mattip)
Date: Sat, 30 Nov 2019 11:30:39 -0800 (PST)
Subject: [pypy-commit] pypy py3.6: merge default into py3.6
Message-ID: <5de2c35f.1c69fb81.8834f.57a9@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r98195:f4a6aa79a6a9
Date: 2019-11-30 21:26 +0200
http://bitbucket.org/pypy/pypy/changeset/f4a6aa79a6a9/

Log:	merge default into py3.6

diff --git a/pypy/module/_minimal_curses/fficurses.py b/pypy/module/_minimal_curses/fficurses.py
--- a/pypy/module/_minimal_curses/fficurses.py
+++ b/pypy/module/_minimal_curses/fficurses.py
@@ -7,8 +7,16 @@
 
 # We cannot trust ncurses5-config, it's broken in various ways in
 # various versions.  For example it might not list -ltinfo even though
-# it's needed, or --cflags might be completely empty.  On Ubuntu 10.04
-# it gives -I/usr/include/ncurses, which doesn't exist at all.  Crap.
+# it's needed, or --cflags might be completely empty. Crap.
+
+IS_CENTOS_6_10 = False
+try:
+    with open('/etc/redhat-release') as fid:
+        for line in fid:
+            if 'CentOS release 6.10' in line:
+                IS_CENTOS_6_10 = True
+except IOError:
+    pass
 
 def try_cflags():
     yield ExternalCompilationInfo(includes=['curses.h', 'term.h'])
@@ -20,8 +28,9 @@
                                             'ncurses/term.h'])
 
 def try_ldflags():
+    yield ExternalCompilationInfo(libraries=['curses', 'tinfo'])
     yield ExternalCompilationInfo(libraries=['curses'])
-    yield ExternalCompilationInfo(libraries=['curses', 'tinfo'])
+    yield ExternalCompilationInfo(libraries=['ncurses', 'tinfo'])
     yield ExternalCompilationInfo(libraries=['ncurses'])
     yield ExternalCompilationInfo(libraries=['ncurses'],
                                   library_dirs=['/usr/lib64'])
@@ -29,6 +38,8 @@
                                   library_dirs=['/usr/lib64'])
 
 def try_tools():
+    if IS_CENTOS_6_10:
+        return
     try:
         yield ExternalCompilationInfo.from_pkg_config("ncurses")
     except Exception:
diff --git a/pypy/module/zlib/test/test_zlib.py b/pypy/module/zlib/test/test_zlib.py
--- a/pypy/module/zlib/test/test_zlib.py
+++ b/pypy/module/zlib/test/test_zlib.py
@@ -403,7 +403,7 @@
 
         assert (d1 + from_copy) == (d1 + from_compressor)
 
-    @py.test.mark.skipif(rzlib.ZLIB_VERSION == '1.2.8', reason='does not error check')
+    @py.test.mark.skipif(rzlib.ZLIB_VERSION in ('1.2.8', '1.2.3'), reason='does not error check')
     def test_cannot_copy_compressor_with_stream_in_inconsistent_state(self):
         if self.runappdirect: skip("can't run with -A")
         compressor = self.zlib.compressobj()
diff --git a/rpython/rlib/rtime.py b/rpython/rlib/rtime.py
--- a/rpython/rlib/rtime.py
+++ b/rpython/rlib/rtime.py
@@ -9,7 +9,7 @@
 from rpython.rtyper.tool import rffi_platform
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib.objectmodel import register_replacement_for
-from rpython.rlib.rarithmetic import intmask, UINT_MAX
+from rpython.rlib.rarithmetic import intmask, r_int64, UINT_MAX
 from rpython.rlib import rposix
 
 _WIN32 = sys.platform.startswith('win')
@@ -94,6 +94,10 @@
     return (float(rffi.getintfield(t, 'c_tv_sec')) +
             float(rffi.getintfield(t, 'c_tv_usec')) * 0.000001)
 
+def decode_timeval_ns(t):
+    return (r_int64(rffi.getintfield(t, 'c_tv_sec')) * 10**9 +
+            r_int64(rffi.getintfield(t, 'c_tv_usec')) * 10**3)
+
 
 def external(name, args, result, compilation_info=eci, **kwds):
     return rffi.llexternal(name, args, result,

From pypy.commits at gmail.com  Sat Nov 30 14:30:41 2019
From: pypy.commits at gmail.com (mattip)
Date: Sat, 30 Nov 2019 11:30:41 -0800 (PST)
Subject: [pypy-commit] pypy py3.6: clean out branches left over from merge
Message-ID: <5de2c361.1c69fb81.fb0e2.a6fb@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r98196:fe543d1b25df
Date: 2019-11-30 21:29 +0200
http://bitbucket.org/pypy/pypy/changeset/fe543d1b25df/

Log:	clean out branches left over from merge

diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -5,15 +5,3 @@
 .. this is the revision after release-pypy3.6-v7.3.0
 .. startrev: 78b4d0a7cf2e
 
-
-.. branch: py3.6-asyncgen
-
-Fix asyncgen_hooks and refactor coroutine execution
-
-.. branch: py3.6-exc-info
-
-Follow CPython's use of exc_info more closely (issue 3096)
-
-.. branch: code_page-utf8
-
-Add encoding, decoding of codepages on windows

From pypy.commits at gmail.com  Sat Nov 30 15:10:06 2019
From: pypy.commits at gmail.com (mattip)
Date: Sat, 30 Nov 2019 12:10:06 -0800 (PST)
Subject: [pypy-commit] pypy backport-decode_timeval_ns-py3.7: close merged
 branch
Message-ID: <5de2cc9e.1c69fb81.9f4ff.f454@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: backport-decode_timeval_ns-py3.7
Changeset: r98197:dddf5797f996
Date: 2019-11-30 21:31 +0200
http://bitbucket.org/pypy/pypy/changeset/dddf5797f996/

Log:	close merged branch


From pypy.commits at gmail.com  Sat Nov 30 15:10:08 2019
From: pypy.commits at gmail.com (mattip)
Date: Sat, 30 Nov 2019 12:10:08 -0800 (PST)
Subject: [pypy-commit] pypy default: merge closed
 backport-decode_timeval_ns-py3.7
Message-ID: <5de2cca0.1c69fb81.f6f6e.02b7@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r98198:bd2b1a5f595d
Date: 2019-11-30 21:31 +0200
http://bitbucket.org/pypy/pypy/changeset/bd2b1a5f595d/

Log:	merge closed backport-decode_timeval_ns-py3.7


From pypy.commits at gmail.com  Sat Nov 30 15:10:10 2019
From: pypy.commits at gmail.com (mattip)
Date: Sat, 30 Nov 2019 12:10:10 -0800 (PST)
Subject: [pypy-commit] pypy default: document merged branch
Message-ID: <5de2cca2.1c69fb81.709cf.c4c9@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r98199:39d8c91deccf
Date: 2019-11-30 21:34 +0200
http://bitbucket.org/pypy/pypy/changeset/39d8c91deccf/

Log:	document merged branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -5,4 +5,6 @@
 .. this is a revision shortly after release-pypy-7.3.0
 .. startrev: dbbbae99135f 
 
+.. branch: backport-decode_timeval_ns-py3.7
 
+Backport ``rtime.decode_timeval_ns`` from py3.7 to rpython

From pypy.commits at gmail.com  Sat Nov 30 15:10:11 2019
From: pypy.commits at gmail.com (mattip)
Date: Sat, 30 Nov 2019 12:10:11 -0800 (PST)
Subject: [pypy-commit] pypy default: 32-bit docker reports
 platform.machine() == 'i686'
Message-ID: <5de2cca3.1c69fb81.e663f.df13@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r98200:f0d91b135c1e
Date: 2019-11-30 22:09 +0200
http://bitbucket.org/pypy/pypy/changeset/f0d91b135c1e/

Log:	32-bit docker reports platform.machine() == 'i686'

diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -4,8 +4,10 @@
 from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC
 from rpython.rlib.rawstorage import misaligned_is_fine
 
+IS_X86 = platform.machine().startswith('x86') or platform.machine() == 'i686'
+
 def no_vector_backend():
-    if platform.machine().startswith('x86'):
+    if IS_X86:
         from rpython.jit.backend.x86.detect_feature import detect_sse4_2
         if sys.maxsize < 2**31:
             return True    
@@ -19,7 +21,7 @@
     return True
 
 def align_check(input):
-    if platform.machine().startswith('x86'):
+    if IS_X86:
         return ""
     if sys.maxsize > 2**32:
         mask = 7

From pypy.commits at gmail.com  Sat Nov 30 23:58:36 2019
From: pypy.commits at gmail.com (mattip)
Date: Sat, 30 Nov 2019 20:58:36 -0800 (PST)
Subject: [pypy-commit] pypy default: only add socket error attributes in
 python2 (issue 3119)
Message-ID: <5de3487c.1c69fb81.7a0e0.3e71@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r98201:f0f95a62d52d
Date: 2019-12-01 06:53 +0200
http://bitbucket.org/pypy/pypy/changeset/f0f95a62d52d/

Log:	only add socket error attributes in python2 (issue 3119)

diff --git a/lib_pypy/_cffi_ssl/_stdssl/error.py b/lib_pypy/_cffi_ssl/_stdssl/error.py
--- a/lib_pypy/_cffi_ssl/_stdssl/error.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/error.py
@@ -27,13 +27,14 @@
         if self.strerror and isinstance(self.strerror, str):
             return self.strerror
         return str(self.args)
-# these are expected on socket as well
-socket.sslerror = SSLError
-for v in [ 'SSL_ERROR_ZERO_RETURN', 'SSL_ERROR_WANT_READ',
-     'SSL_ERROR_WANT_WRITE', 'SSL_ERROR_WANT_X509_LOOKUP', 'SSL_ERROR_SYSCALL',
-     'SSL_ERROR_SSL', 'SSL_ERROR_WANT_CONNECT', 'SSL_ERROR_EOF',
-     'SSL_ERROR_INVALID_ERROR_CODE' ]:
-    setattr(socket, v, locals()[v]) 
+# these are expected on socket in python2 as well
+if sys.version_info[0] < 3:
+    socket.sslerror = SSLError
+    for v in [ 'SSL_ERROR_ZERO_RETURN', 'SSL_ERROR_WANT_READ',
+         'SSL_ERROR_WANT_WRITE', 'SSL_ERROR_WANT_X509_LOOKUP', 'SSL_ERROR_SYSCALL',
+         'SSL_ERROR_SSL', 'SSL_ERROR_WANT_CONNECT', 'SSL_ERROR_EOF',
+         'SSL_ERROR_INVALID_ERROR_CODE' ]:
+        setattr(socket, v, locals()[v]) 
 
 class SSLZeroReturnError(SSLError):
     """ SSL/TLS session closed cleanly. """

From pypy.commits at gmail.com  Sat Nov 30 23:58:38 2019
From: pypy.commits at gmail.com (mattip)
Date: Sat, 30 Nov 2019 20:58:38 -0800 (PST)
Subject: [pypy-commit] pypy py3.6: merge default into py3.6
Message-ID: <5de3487e.1c69fb81.44bec.691e@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r98202:5570fbfd5401
Date: 2019-12-01 06:57 +0200
http://bitbucket.org/pypy/pypy/changeset/5570fbfd5401/

Log:	merge default into py3.6

diff --git a/lib_pypy/_cffi_ssl/_stdssl/error.py b/lib_pypy/_cffi_ssl/_stdssl/error.py
--- a/lib_pypy/_cffi_ssl/_stdssl/error.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/error.py
@@ -27,13 +27,14 @@
         if self.strerror and isinstance(self.strerror, str):
             return self.strerror
         return str(self.args)
-# these are expected on socket as well
-socket.sslerror = SSLError
-for v in [ 'SSL_ERROR_ZERO_RETURN', 'SSL_ERROR_WANT_READ',
-     'SSL_ERROR_WANT_WRITE', 'SSL_ERROR_WANT_X509_LOOKUP', 'SSL_ERROR_SYSCALL',
-     'SSL_ERROR_SSL', 'SSL_ERROR_WANT_CONNECT', 'SSL_ERROR_EOF',
-     'SSL_ERROR_INVALID_ERROR_CODE' ]:
-    setattr(socket, v, locals()[v]) 
+# these are expected on socket in python2 as well
+if sys.version_info[0] < 3:
+    socket.sslerror = SSLError
+    for v in [ 'SSL_ERROR_ZERO_RETURN', 'SSL_ERROR_WANT_READ',
+         'SSL_ERROR_WANT_WRITE', 'SSL_ERROR_WANT_X509_LOOKUP', 'SSL_ERROR_SYSCALL',
+         'SSL_ERROR_SSL', 'SSL_ERROR_WANT_CONNECT', 'SSL_ERROR_EOF',
+         'SSL_ERROR_INVALID_ERROR_CODE' ]:
+        setattr(socket, v, locals()[v]) 
 
 class SSLZeroReturnError(SSLError):
     """ SSL/TLS session closed cleanly. """
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -5,4 +5,6 @@
 .. this is a revision shortly after release-pypy-7.3.0
 .. startrev: dbbbae99135f 
 
+.. branch: backport-decode_timeval_ns-py3.7
 
+Backport ``rtime.decode_timeval_ns`` from py3.7 to rpython
diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -4,8 +4,10 @@
 from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC
 from rpython.rlib.rawstorage import misaligned_is_fine
 
+IS_X86 = platform.machine().startswith('x86') or platform.machine() == 'i686'
+
 def no_vector_backend():
-    if platform.machine().startswith('x86'):
+    if IS_X86:
         from rpython.jit.backend.x86.detect_feature import detect_sse4_2
         if sys.maxsize < 2**31:
             return True    
@@ -19,7 +21,7 @@
     return True
 
 def align_check(input):
-    if platform.machine().startswith('x86'):
+    if IS_X86:
         return ""
     if sys.maxsize > 2**32:
         mask = 7